isomorfeus-ferret 0.12.6 → 0.12.7

data/ext/isomorfeus_ferret_ext/frt_index.c

@@ -6,6 +6,8 @@
 #include <string.h>
 #include <limits.h>
 #include <ctype.h>
+#include "brotli_decode.h"
+#include "brotli_encode.h"
 
 extern void frt_micro_sleep(const int micro_seconds);
 
@@ -39,8 +41,8 @@ static char *ste_next(FrtTermEnum *te);
 #define FORMAT 0
 #define SEGMENTS_GEN_FILE_NAME "segments"
 #define MAX_EXT_LEN 10
-#define ZIP_BUFFER_SIZE 16348
-#define ZIP_LEVEL 9
+#define COMPRESSION_BUFFER_SIZE 16348
+#define COMPRESSION_LEVEL 9
 
 /* *** Must be three characters *** */
 static const char *INDEX_EXTENSIONS[] = {
@@ -220,6 +222,9 @@ static void fi_set_store(FrtFieldInfo *fi, int store)
         case FRT_STORE_YES:
             fi->bits |= FRT_FI_IS_STORED_BM;
             break;
+        case FRT_STORE_COMPRESS:
+            fi->bits |= FRT_FI_IS_COMPRESSED_BM | FRT_FI_IS_STORED_BM;
+            break;
     }
 }
 
@@ -304,8 +309,9 @@ char *frt_fi_to_s(FrtFieldInfo *fi)
     const char *fi_name = rb_id2name(fi->name);
     char *str = FRT_ALLOC_N(char, strlen(fi_name) + 200);
     char *s = str;
-    s += sprintf(str, "[\"%s\":(%s%s%s%s%s%s%s", fi_name,
+    s += sprintf(str, "[\"%s\":(%s%s%s%s%s%s%s%s", fi_name,
                  fi_is_stored(fi) ? "is_stored, " : "",
+                 fi_is_compressed(fi) ? "is_compressed, " : "",
                  fi_is_indexed(fi) ? "is_indexed, " : "",
                  fi_is_tokenized(fi) ? "is_tokenized, " : "",
                  fi_omit_norms(fi) ? "omit_norms, " : "",
@@ -443,7 +449,8 @@ void frt_fis_write(FrtFieldInfos *fis, FrtOutStream *os)
 static const char *store_str[] = {
     ":no",
     ":yes",
-    ""
+    "",
+    ":compressed"
 };
 
 static const char *fi_store_str(FrtFieldInfo *fi)
@@ -1145,12 +1152,13 @@ frt_u64 frt_sis_read_current_version(FrtStore *store)
  *
  ****************************************************************************/
 
-static FrtLazyDocField *lazy_df_new(FrtSymbol name, const int size)
+static FrtLazyDocField *lazy_df_new(FrtSymbol name, const int size, bool is_compressed)
 {
     FrtLazyDocField *self = FRT_ALLOC(FrtLazyDocField);
     self->name = name;
     self->size = size;
     self->data = FRT_ALLOC_AND_ZERO_N(FrtLazyDocFieldData, size);
+    self->is_compressed = is_compressed;
     return self;
 }
 
@@ -1166,6 +1174,52 @@ static void lazy_df_destroy(FrtLazyDocField *self)
     free(self);
 }
 
+static void comp_raise()
+{
+    FRT_RAISE(EXCEPTION, "Compression error");
+}
+
+static char *is_read_compressed_bytes(FrtInStream *is, int compressed_len, int *len)
+{
+    int buf_out_idx = 0;
+    int read_len;
+    frt_uchar buf_in[COMPRESSION_BUFFER_SIZE];
+    const frt_uchar *next_in;
+    size_t available_in;
+    frt_uchar *buf_out = NULL;
+    frt_uchar *next_out;
+    size_t available_out;
+
+    BrotliDecoderState *b_state = BrotliDecoderCreateInstance(NULL, NULL, NULL);
+    BrotliDecoderResult b_result = BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT;
+    if (!b_state) { comp_raise(); return NULL; }
+
+    do {
+        read_len = compressed_len > COMPRESSION_BUFFER_SIZE ? COMPRESSION_BUFFER_SIZE : compressed_len;
+        frt_is_read_bytes(is, buf_in, read_len);
+        compressed_len -= read_len;
+        available_in = read_len;
+        next_in = buf_in;
+        available_out = COMPRESSION_BUFFER_SIZE;
+        do {
+            FRT_REALLOC_N(buf_out, frt_uchar, buf_out_idx + COMPRESSION_BUFFER_SIZE);
+            next_out = buf_out + buf_out_idx;
+            b_result = BrotliDecoderDecompressStream(b_state,
+                &available_in, &next_in,
+                &available_out, &next_out, NULL);
+            if (b_result == BROTLI_DECODER_RESULT_ERROR) { comp_raise(); return NULL; }
+            buf_out_idx += COMPRESSION_BUFFER_SIZE - available_out;
+        } while (b_result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT);
+    } while (b_result != BROTLI_DECODER_RESULT_SUCCESS && compressed_len > 0);
+
+    BrotliDecoderDestroyInstance(b_state);
+
+    FRT_REALLOC_N(buf_out, frt_uchar, buf_out_idx + 1);
+    buf_out[buf_out_idx] = '\0';
+    *len = buf_out_idx;
+    return (char *)buf_out;
+}
+
 char *frt_lazy_df_get_data(FrtLazyDocField *self, int i)
 {
     char *text = NULL;
@@ -1174,9 +1228,13 @@ char *frt_lazy_df_get_data(FrtLazyDocField *self, int i)
         if (NULL == text) {
             const int read_len = self->data[i].length + 1;
             frt_is_seek(self->doc->fields_in, self->data[i].start);
-            self->data[i].text = text = FRT_ALLOC_N(char, read_len);
-            frt_is_read_bytes(self->doc->fields_in, (frt_uchar *)text, read_len);
-            text[read_len - 1] = '\0';
+            if (self->is_compressed) {
+                self->data[i].text = text = is_read_compressed_bytes(self->doc->fields_in, read_len, &(self->data[i].length));
+            } else {
+                self->data[i].text = text = FRT_ALLOC_N(char, read_len);
+                frt_is_read_bytes(self->doc->fields_in, (frt_uchar *)text, read_len);
+                text[read_len - 1] = '\0';
+            }
         }
     }
 
@@ -1185,6 +1243,16 @@ char *frt_lazy_df_get_data(FrtLazyDocField *self, int i)
 
 void frt_lazy_df_get_bytes(FrtLazyDocField *self, char *buf, int start, int len)
 {
+    if (self->is_compressed == 1) {
+        int i;
+        self->len = 0;
+        for (i = self->size-1; i >= 0; i--) {
+            (void)frt_lazy_df_get_data(self, i);
+            self->len += self->data[i].length + 1;
+        }
+        self->len--; /* each field separated by ' ' but no need to add to end */
+        self->is_compressed = 2;
+    }
     if (start < 0 || start >= self->len) {
         FRT_RAISE(FRT_IO_ERROR, "start out of range in LazyDocField#get_bytes. %d "
               "is not between 0 and %d", start, self->len);
@@ -1196,7 +1264,33 @@ void frt_lazy_df_get_bytes(FrtLazyDocField *self, char *buf, int start, int len)
         FRT_RAISE(FRT_IO_ERROR, "Tried to read past end of field. Field is only %d "
               "bytes long but tried to read to %d", self->len, start + len);
     }
-    else {
+    if (self->is_compressed) {
+        int cur_start = 0, buf_start = 0, cur_end, i, copy_start, copy_len;
+        for (i = 0; i < self->size; i++) {
+            cur_end = cur_start + self->data[i].length;
+            if (start < cur_end) {
+                copy_start = start > cur_start ? start - cur_start : 0;
+                copy_len = cur_end - cur_start - copy_start;
+                if (copy_len >= len) {
+                    copy_len = len;
+                    len = 0;
+                }
+                else {
+                    len -= copy_len;
+                }
+                memcpy(buf + buf_start,
+                       self->data[i].text + copy_start,
+                       copy_len);
+                buf_start += copy_len;
+                if (len > 0) {
+                    buf[buf_start++] = ' ';
+                    len--;
+                }
+                if (len == 0) break;
+            }
+            cur_start = cur_end + 1;
+        }
+    } else {
         frt_is_seek(self->doc->fields_in, self->data[0].start + start);
         frt_is_read_bytes(self->doc->fields_in, (frt_uchar *)buf, len);
     }
@@ -1286,7 +1380,7 @@ void frt_fr_close(FrtFieldsReader *fr)
     free(fr);
 }
 
-static FrtDocField *frt_fr_df_new(FrtSymbol name, int size)
+static FrtDocField *frt_fr_df_new(FrtSymbol name, int size, bool is_compressed)
 {
     FrtDocField *df = FRT_ALLOC(FrtDocField);
     df->name = name;
@@ -1295,9 +1389,22 @@ static FrtDocField *frt_fr_df_new(FrtSymbol name, int size)
     df->lengths = FRT_ALLOC_N(int, df->capa);
     df->destroy_data = true;
     df->boost = 1.0f;
+    df->is_compressed = is_compressed;
     return df;
 }
 
+static void frt_fr_read_compressed_fields(FrtFieldsReader *fr, FrtDocField *df)
+{
+    int i;
+    const int df_size = df->size;
+    FrtInStream *fdt_in = fr->fdt_in;
+
+    for (i = 0; i < df_size; i++) {
+        const int compressed_len = df->lengths[i] + 1;
+        df->data[i] = is_read_compressed_bytes(fdt_in, compressed_len, &(df->lengths[i]));
+    }
+}
+
 FrtDocument *frt_fr_get_doc(FrtFieldsReader *fr, int doc_num)
 {
     int i, j;
@@ -1316,7 +1423,7 @@ FrtDocument *frt_fr_get_doc(FrtFieldsReader *fr, int doc_num)
         const int field_num = frt_is_read_vint(fdt_in);
         FrtFieldInfo *fi = fr->fis->fields[field_num];
         const int df_size = frt_is_read_vint(fdt_in);
-        FrtDocField *df = frt_fr_df_new(fi->name, df_size);
+        FrtDocField *df = frt_fr_df_new(fi->name, df_size, fi_is_compressed(fi));
 
         for (j = 0; j < df_size; j++) {
             df->lengths[j] = frt_is_read_vint(fdt_in);
@@ -1326,12 +1433,16 @@ FrtDocument *frt_fr_get_doc(FrtFieldsReader *fr, int doc_num)
     }
     for (i = 0; i < stored_cnt; i++) {
         FrtDocField *df = doc->fields[i];
-        const int df_size = df->size;
-        for (j = 0; j < df_size; j++) {
-            const int read_len = df->lengths[j] + 1;
-            df->data[j] = FRT_ALLOC_N(char, read_len);
-            frt_is_read_bytes(fdt_in, (frt_uchar *)df->data[j], read_len);
-            df->data[j][read_len - 1] = '\0';
+        if (df->is_compressed) {
+            frt_fr_read_compressed_fields(fr, df);
+        } else {
+            const int df_size = df->size;
+            for (j = 0; j < df_size; j++) {
+                const int read_len = df->lengths[j] + 1;
+                df->data[j] = FRT_ALLOC_N(char, read_len);
+                frt_is_read_bytes(fdt_in, (frt_uchar *)df->data[j], read_len);
+                df->data[j][read_len - 1] = '\0';
+            }
         }
     }
 
@@ -1355,7 +1466,7 @@ FrtLazyDoc *frt_fr_get_lazy_doc(FrtFieldsReader *fr, int doc_num)
     for (i = 0; i < stored_cnt; i++) {
         FrtFieldInfo *fi = fr->fis->fields[frt_is_read_vint(fdt_in)];
         const int data_cnt = frt_is_read_vint(fdt_in);
-        FrtLazyDocField *lazy_df = lazy_df_new(fi->name, data_cnt);
+        FrtLazyDocField *lazy_df = lazy_df_new(fi->name, data_cnt, fi_is_compressed(fi));
         const int field_start = start;
         /* get the starts relative positions this time around */
         for (j = 0; j < data_cnt; j++) {
@@ -1549,6 +1660,37 @@ void frt_fw_close(FrtFieldsWriter *fw)
     free(fw);
 }
 
+static int frt_os_write_compressed_bytes(FrtOutStream* out_stream, frt_uchar *data, int length)
+{
+    size_t compressed_len = 0;
+    const frt_uchar *next_in = data;
+    size_t available_in = length;
+    size_t available_out;
+    frt_uchar compression_buffer[COMPRESSION_BUFFER_SIZE];
+    frt_uchar *next_out;
+    BrotliEncoderState *b_state = BrotliEncoderCreateInstance(NULL, NULL, NULL);
+    if (!b_state) { comp_raise(); return -1; }
+
+    BrotliEncoderSetParameter(b_state, BROTLI_PARAM_QUALITY, COMPRESSION_LEVEL);
+
+    do {
+        available_out = COMPRESSION_BUFFER_SIZE;
+        next_out = compression_buffer;
+        if (!BrotliEncoderCompressStream(b_state, BROTLI_OPERATION_FINISH,
+            &available_in, &next_in,
+            &available_out, &next_out, &compressed_len)) {
+            BrotliEncoderDestroyInstance(b_state);
+            comp_raise();
+            return -1;
+        }
+        frt_os_write_bytes(out_stream, compression_buffer, COMPRESSION_BUFFER_SIZE - available_out);
+    } while (!BrotliEncoderIsFinished(b_state));
+
+    BrotliEncoderDestroyInstance(b_state);
+    // fprintf(stderr, "Compressed: %i -> %i\n", length, (int)compressed_len);
+    return (int)compressed_len;
+}
+
 void frt_fw_add_doc(FrtFieldsWriter *fw, FrtDocument *doc)
 {
     int i, j, stored_cnt = 0;
@@ -1577,13 +1719,20 @@ void frt_fw_add_doc(FrtFieldsWriter *fw, FrtDocument *doc)
             const int df_size = df->size;
             frt_os_write_vint(fdt_out, fi->number);
             frt_os_write_vint(fdt_out, df_size);
-            for (j = 0; j < df_size; j++) {
-                const int length = df->lengths[j];
-                frt_os_write_vint(fdt_out, length);
-                frt_os_write_bytes(fw->buffer, (frt_uchar*)df->data[j], length);
-                /* leave a space between fields as that is how they are
-                    * analyzed */
-                frt_os_write_byte(fw->buffer, ' ');
+            if (fi_is_compressed(fi)) {
+                for (j = 0; j < df_size; j++) {
+                    const int length = df->lengths[j];
+                    int compressed_len = frt_os_write_compressed_bytes(fw->buffer, (frt_uchar*)df->data[j], length);
+                    frt_os_write_vint(fdt_out, compressed_len - 1);
+                }
+            } else {
+                for (j = 0; j < df_size; j++) {
+                    const int length = df->lengths[j];
+                    frt_os_write_vint(fdt_out, length);
+                    frt_os_write_bytes(fw->buffer, (frt_uchar*)df->data[j], length);
+                    /* leave a space between fields as that is how they are analyzed */
+                    frt_os_write_byte(fw->buffer, ' ');
+                }
             }
         }
     }

data/ext/isomorfeus_ferret_ext/frt_index.h

@@ -66,7 +66,8 @@ extern FrtHash *frt_co_hash_create();
 typedef enum
 {
     FRT_STORE_NO = 0,
-    FRT_STORE_YES = 1
+    FRT_STORE_YES = 1,
+    FRT_STORE_COMPRESS = 2
 } FrtStoreValue;
 
 typedef enum
@@ -88,6 +89,7 @@ typedef enum
 } FrtTermVectorValue;
 
 #define FRT_FI_IS_STORED_BM         0x001
+#define FRT_FI_IS_COMPRESSED_BM     0x002
 #define FRT_FI_IS_INDEXED_BM        0x004
 #define FRT_FI_IS_TOKENIZED_BM      0x008
 #define FRT_FI_OMIT_NORMS_BM        0x010
@@ -112,6 +114,7 @@ extern char *frt_fi_to_s(FrtFieldInfo *fi);
 extern void frt_fi_deref(FrtFieldInfo *fi);
 
 #define fi_is_stored(fi)         (((fi)->bits & FRT_FI_IS_STORED_BM) != 0)
+#define fi_is_compressed(fi)     (((fi)->bits & FRT_FI_IS_COMPRESSED_BM) != 0)
 #define fi_is_indexed(fi)        (((fi)->bits & FRT_FI_IS_INDEXED_BM) != 0)
 #define fi_is_tokenized(fi)      (((fi)->bits & FRT_FI_IS_TOKENIZED_BM) != 0)
 #define fi_omit_norms(fi)        (((fi)->bits & FRT_FI_OMIT_NORMS_BM) != 0)
@@ -575,11 +578,11 @@ typedef struct FrtLazyDocField
     FrtLazyDoc          *doc;
     int                 size; /* number of data elements */
     int                 len;  /* length of data elements concatenated */
+    int                 is_compressed : 2; /* set to 2 after all data is loaded */
 } FrtLazyDocField;
 
 extern char *frt_lazy_df_get_data(FrtLazyDocField *self, int i);
-extern void frt_lazy_df_get_bytes(FrtLazyDocField *self, char *buf,
-                              int start, int len);
+extern void frt_lazy_df_get_bytes(FrtLazyDocField *self, char *buf, int start, int len);
 
 /* * * FrtLazyDoc * * */
 struct FrtLazyDoc

data/ext/isomorfeus_ferret_ext/frt_ram_store.c

@@ -179,6 +179,7 @@ static off_t ram_length(FrtStore *store, const char *filename)
 
 static void ramo_flush_i(FrtOutStream *os, const frt_uchar *src, int len)
 {
+    if (len == 0) { return; }
     frt_uchar *buffer;
     FrtRAMFile *rf = os->file.rf;
     int buffer_number, buffer_offset, bytes_in_buffer, bytes_to_copy;

data/ext/isomorfeus_ferret_ext/test_fields.c

@@ -2,10 +2,10 @@
 #include "test.h"
 
 #define do_field_prop_test(tc, fi, name, boost, is_stored,\
-                           is_indexed, is_tokenized, omit_norms,\
+                           is_compressed, is_indexed, is_tokenized, omit_norms,\
                            store_term_vector, store_positions, store_offsets)\
         field_prop_test(tc, __LINE__, fi, name, boost, is_stored,\
-                        is_indexed, is_tokenized, omit_norms,\
+                        is_compressed, is_indexed, is_tokenized, omit_norms,\
                         store_term_vector, store_positions, store_offsets)
 #define T 1
 #define F 0
@@ -16,6 +16,7 @@ void field_prop_test(TestCase *tc,
                      FrtSymbol name,
                      float boost,
                      bool is_stored,
+                     bool is_compressed,
                      bool is_indexed,
                      bool is_tokenized,
                      bool omit_norms,
@@ -26,6 +27,7 @@ void field_prop_test(TestCase *tc,
     tst_ptr_equal(line_num, tc, (void *)name, (void *)fi->name);
     tst_flt_equal(line_num, tc, boost, fi->boost);
     tst_int_equal(line_num, tc, is_stored,          fi_is_stored(fi));
+    tst_int_equal(line_num, tc, is_compressed,      fi_is_compressed(fi));
     tst_int_equal(line_num, tc, is_indexed,         fi_is_indexed(fi));
     tst_int_equal(line_num, tc, is_tokenized,       fi_is_tokenized(fi));
     tst_int_equal(line_num, tc, omit_norms,         fi_omit_norms(fi));
@@ -45,17 +47,20 @@ static void test_fi_new(TestCase *tc, void *data)
     FrtFieldInfo *fi;
     (void)data; /* suppress unused argument warning */
     fi = frt_fi_new(rb_intern("name"), FRT_STORE_NO, FRT_INDEX_NO, FRT_TERM_VECTOR_NO);
-    do_field_prop_test(tc, fi, rb_intern("name"), 1.0, F, F, F, F, F, F, F);
+    do_field_prop_test(tc, fi, rb_intern("name"), 1.0, F, F, F, F, F, F, F, F);
     frt_fi_deref(fi);
     fi = frt_fi_new(rb_intern("name"), FRT_STORE_YES, FRT_INDEX_YES, FRT_TERM_VECTOR_YES);
-    do_field_prop_test(tc, fi, rb_intern("name"), 1.0, T, T, T, F, T, F, F);
+    do_field_prop_test(tc, fi, rb_intern("name"), 1.0, T, F, T, T, F, T, F, F);
+    frt_fi_deref(fi);
+    fi = frt_fi_new(rb_intern("name"), FRT_STORE_COMPRESS, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_POSITIONS);
+    do_field_prop_test(tc, fi, rb_intern("name"), 1.0, T, T, T, F, F, T, T, F);
     frt_fi_deref(fi);
     fi = frt_fi_new(rb_intern("name"), FRT_STORE_NO, FRT_INDEX_YES_OMIT_NORMS, FRT_TERM_VECTOR_WITH_OFFSETS);
-    do_field_prop_test(tc, fi, rb_intern("name"), 1.0, F, T, T, T, T, F, T);
+    do_field_prop_test(tc, fi, rb_intern("name"), 1.0, F, F, T, T, T, T, F, T);
     frt_fi_deref(fi);
     fi = frt_fi_new(rb_intern("name"), FRT_STORE_NO, FRT_INDEX_UNTOKENIZED_OMIT_NORMS, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
     fi->boost = 1000.0;
-    do_field_prop_test(tc, fi, rb_intern("name"), 1000.0, F, T, F, T, T, T, T);
+    do_field_prop_test(tc, fi, rb_intern("name"), 1000.0, F, F, T, F, T, T, T, T);
     frt_fi_deref(fi);
 }
 
@@ -75,7 +80,7 @@ static void test_fis_basic(TestCase *tc, void *data)
     fis = frt_fis_new(FRT_STORE_NO, FRT_INDEX_NO, FRT_TERM_VECTOR_NO);
     frt_fis_add_field(fis, frt_fi_new(rb_intern("FFFFFFFF"), FRT_STORE_NO, FRT_INDEX_NO, FRT_TERM_VECTOR_NO));
     frt_fis_add_field(fis, frt_fi_new(rb_intern("TFTTFTFF"), FRT_STORE_YES, FRT_INDEX_YES, FRT_TERM_VECTOR_YES));
-    frt_fis_add_field(fis, frt_fi_new(rb_intern("TTTFFTTF"), FRT_STORE_YES, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_POSITIONS));
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("TTTFFTTF"), FRT_STORE_COMPRESS, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_POSITIONS));
     frt_fis_add_field(fis, frt_fi_new(rb_intern("FFTTTTFT"), FRT_STORE_NO, FRT_INDEX_YES_OMIT_NORMS, FRT_TERM_VECTOR_WITH_OFFSETS));
     frt_fis_add_field(fis, frt_fi_new(rb_intern("FFTFTTTT"), FRT_STORE_NO, FRT_INDEX_UNTOKENIZED_OMIT_NORMS, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS));
 
@@ -113,11 +118,11 @@ static void test_fis_basic(TestCase *tc, void *data)
     fis->fields[3]->boost = 4.0;
     fis->fields[4]->boost = 5.0;
 
-    do_field_prop_test(tc, fis->fields[0], rb_intern("FFFFFFFF"), 1.0, F, F, F, F, F, F, F);
-    do_field_prop_test(tc, fis->fields[1], rb_intern("TFTTFTFF"), 2.0, T, T, T, F, T, F, F);
-    do_field_prop_test(tc, fis->fields[2], rb_intern("TTTFFTTF"), 3.0, T, T, F, F, T, T, F);
-    do_field_prop_test(tc, fis->fields[3], rb_intern("FFTTTTFT"), 4.0, F, T, T, T, T, F, T);
-    do_field_prop_test(tc, fis->fields[4], rb_intern("FFTFTTTT"), 5.0, F, T, F, T, T, T, T);
+    do_field_prop_test(tc, fis->fields[0], rb_intern("FFFFFFFF"), 1.0, F, F, F, F, F, F, F, F);
+    do_field_prop_test(tc, fis->fields[1], rb_intern("TFTTFTFF"), 2.0, T, F, T, T, F, T, F, F);
+    do_field_prop_test(tc, fis->fields[2], rb_intern("TTTFFTTF"), 3.0, T, T, T, F, F, T, T, F);
+    do_field_prop_test(tc, fis->fields[3], rb_intern("FFTTTTFT"), 4.0, F, F, T, T, T, T, F, T);
+    do_field_prop_test(tc, fis->fields[4], rb_intern("FFTFTTTT"), 5.0, F, F, T, F, T, T, T, T);
 
     frt_fis_deref(fis);
 }
@@ -128,26 +133,26 @@ static void test_fis_with_default(TestCase *tc, void *data)
     (void)data; /* suppress unused argument warning */
 
     fis = frt_fis_new(FRT_STORE_NO, FRT_INDEX_NO, FRT_TERM_VECTOR_NO);
-    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("name")), rb_intern("name"), 1.0, F, F, F, F, F, F, F);
-    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("dave")), rb_intern("dave"), 1.0, F, F, F, F, F, F, F);
-    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("wert")), rb_intern("wert"), 1.0, F, F, F, F, F, F, F);
-    do_field_prop_test(tc, fis->fields[0], rb_intern("name"), 1.0, F, F, F, F, F, F, F);
-    do_field_prop_test(tc, fis->fields[1], rb_intern("dave"), 1.0, F, F, F, F, F, F, F);
-    do_field_prop_test(tc, fis->fields[2], rb_intern("wert"), 1.0, F, F, F, F, F, F, F);
+    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("name")), rb_intern("name"), 1.0, F, F, F, F, F, F, F, F);
+    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("dave")), rb_intern("dave"), 1.0, F, F, F, F, F, F, F, F);
+    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("wert")), rb_intern("wert"), 1.0, F, F, F, F, F, F, F, F);
+    do_field_prop_test(tc, fis->fields[0], rb_intern("name"), 1.0, F, F, F, F, F, F, F, F);
+    do_field_prop_test(tc, fis->fields[1], rb_intern("dave"), 1.0, F, F, F, F, F, F, F, F);
+    do_field_prop_test(tc, fis->fields[2], rb_intern("wert"), 1.0, F, F, F, F, F, F, F, F);
     Apnull(frt_fis_get_field(fis, rb_intern("random")));
     frt_fis_deref(fis);
 
     fis = frt_fis_new(FRT_STORE_YES, FRT_INDEX_YES, FRT_TERM_VECTOR_YES);
-    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("name")), rb_intern("name"), 1.0, T, T, T, F, T, F, F);
+    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("name")), rb_intern("name"), 1.0, T, F, T, T, F, T, F, F);
     frt_fis_deref(fis);
-    fis = frt_fis_new(FRT_STORE_YES, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_POSITIONS);
-    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("name")), rb_intern("name"), 1.0, T, T, F, F, T, T, F);
+    fis = frt_fis_new(FRT_STORE_COMPRESS, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_POSITIONS);
+    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("name")), rb_intern("name"), 1.0, T, T, T, F, F, T, T, F);
     frt_fis_deref(fis);
     fis = frt_fis_new(FRT_STORE_NO, FRT_INDEX_YES_OMIT_NORMS, FRT_TERM_VECTOR_WITH_OFFSETS);
-    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("name")), rb_intern("name"), 1.0, F, T, T, T, T, F, T);
+    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("name")), rb_intern("name"), 1.0, F, F, T, T, T, T, F, T);
     frt_fis_deref(fis);
     fis = frt_fis_new(FRT_STORE_NO, FRT_INDEX_UNTOKENIZED_OMIT_NORMS, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
-    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("name")), rb_intern("name"), 1.0, F, T, F, T, T, T, T);
+    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("name")), rb_intern("name"), 1.0, F, F, T, F, T, T, T, T);
     frt_fis_deref(fis);
 }
 
@@ -164,22 +169,23 @@ static void test_fis_rw(TestCase *tc, void *data)
                   FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
     frt_fis_add_field(fis, frt_fi_new(rb_intern("FFFFFFFF"), FRT_STORE_NO, FRT_INDEX_NO, FRT_TERM_VECTOR_NO));
     frt_fis_add_field(fis, frt_fi_new(rb_intern("TFTTFTFF"), FRT_STORE_YES, FRT_INDEX_YES, FRT_TERM_VECTOR_YES));
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("TTTFFTTF"), FRT_STORE_COMPRESS, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_POSITIONS));
     frt_fis_add_field(fis, frt_fi_new(rb_intern("FFTTTTFT"), FRT_STORE_NO, FRT_INDEX_YES_OMIT_NORMS, FRT_TERM_VECTOR_WITH_OFFSETS));
     frt_fis_add_field(fis, frt_fi_new(rb_intern("FFTFTTTT"), FRT_STORE_NO, FRT_INDEX_UNTOKENIZED_OMIT_NORMS, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS));
-    fis->fields[0]->boost = 2.0;
-    fis->fields[1]->boost = 3.0;
-    fis->fields[2]->boost = 4.0;
-    fis->fields[3]->boost = 5.0;
+    fis->fields[1]->boost = 2.0;
+    fis->fields[2]->boost = 3.0;
+    fis->fields[3]->boost = 4.0;
+    fis->fields[4]->boost = 5.0;
     os = store->new_output(store, "fields");
     frt_fis_write(fis, os);
     frt_os_close(os);
 
     /* these fields won't be saved be will added again later */
-    Aiequal(4, fis->size);
-    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("new_field")), rb_intern("new_field"), 1.0, T, T, F, T, T, T, T);
     Aiequal(5, fis->size);
-    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("another")), rb_intern("another"), 1.0, T, T, F, T, T, T, T);
+    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("new_field")), rb_intern("new_field"), 1.0, T, F, T, F, T, T, T, T);
     Aiequal(6, fis->size);
+    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("another")), rb_intern("another"), 1.0, T, F, T, F, T, T, T, T);
+    Aiequal(7, fis->size);
 
     frt_fis_deref(fis);
 
@@ -190,15 +196,16 @@ static void test_fis_rw(TestCase *tc, void *data)
     Aiequal(FRT_INDEX_UNTOKENIZED_OMIT_NORMS, fis->index);
     Aiequal(FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS, fis->term_vector);
 
-    do_field_prop_test(tc, fis->fields[0], rb_intern("FFFFFFFF"), 2.0, F, F, F, F, F, F, F);
-    do_field_prop_test(tc, fis->fields[1], rb_intern("TFTTFTFF"), 3.0, T, T, T, F, T, F, F);
-    do_field_prop_test(tc, fis->fields[2], rb_intern("FFTTTTFT"), 4.0, F, T, T, T, T, F, T);
-    do_field_prop_test(tc, fis->fields[3], rb_intern("FFTFTTTT"), 5.0, F, T, F, T, T, T, T);
-    Aiequal(4, fis->size);
-    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("new_field")), rb_intern("new_field"), 1.0, T, T, F, T, T, T, T);
+    do_field_prop_test(tc, fis->fields[0], rb_intern("FFFFFFFF"), 1.0, F, F, F, F, F, F, F, F);
+    do_field_prop_test(tc, fis->fields[1], rb_intern("TFTTFTFF"), 2.0, T, F, T, T, F, T, F, F);
+    do_field_prop_test(tc, fis->fields[2], rb_intern("TTTFFTTF"), 3.0, T, T, T, F, F, T, T, F);
+    do_field_prop_test(tc, fis->fields[3], rb_intern("FFTTTTFT"), 4.0, F, F, T, T, T, T, F, T);
+    do_field_prop_test(tc, fis->fields[4], rb_intern("FFTFTTTT"), 5.0, F, F, T, F, T, T, T, T);
     Aiequal(5, fis->size);
-    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("another")), rb_intern("another"), 1.0, T, T, F, T, T, T, T);
+    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("new_field")), rb_intern("new_field"), 1.0, T, F, T, F, T, T, T, T);
     Aiequal(6, fis->size);
+    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("another")), rb_intern("another"), 1.0, T, F, T, F, T, T, T, T);
+    Aiequal(7, fis->size);
     str = frt_fis_to_s(fis);
     Asequal("default:\n"
             "  store: :yes\n"
@@ -206,15 +213,20 @@ static void test_fis_rw(TestCase *tc, void *data)
             "  term_vector: :with_positions_offsets\n"
             "fields:\n"
             "  FFFFFFFF:\n"
-            "    boost: 2.000000\n"
+            "    boost: 1.000000\n"
             "    store: :no\n"
             "    index: :no\n"
             "    term_vector: :no\n"
             "  TFTTFTFF:\n"
-            "    boost: 3.000000\n"
+            "    boost: 2.000000\n"
             "    store: :yes\n"
             "    index: :yes\n"
             "    term_vector: :yes\n"
+            "  TTTFFTTF:\n"
+            "    boost: 3.000000\n"
+            "    store: :compressed\n"
+            "    index: :untokenized\n"
+            "    term_vector: :with_positions\n"
             "  FFTTTTFT:\n"
             "    boost: 4.000000\n"
             "    store: :no\n"
@@ -302,7 +314,7 @@ static FrtFieldInfos *prepare_fis()
     frt_fis_add_field(fis, frt_fi_new(rb_intern("ignored"), FRT_STORE_NO, FRT_INDEX_NO, FRT_TERM_VECTOR_NO));
     frt_fis_add_field(fis, frt_fi_new(rb_intern("unstored"), FRT_STORE_NO, FRT_INDEX_YES, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS));
     frt_fis_add_field(fis, frt_fi_new(rb_intern("stored"), FRT_STORE_YES, FRT_INDEX_YES, FRT_TERM_VECTOR_YES));
-    frt_fis_add_field(fis, frt_fi_new(rb_intern("stored_array"), FRT_STORE_YES, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_NO));
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("stored_array"), FRT_STORE_COMPRESS, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_NO));
     return fis;
 }
 
@@ -327,8 +339,8 @@ static void test_fields_rw_single(TestCase *tc, void *data)
    frt_doc_destroy(doc);
 
     Aiequal(6, fis->size);
-    do_field_prop_test(tc, frt_fis_get_field(fis, rb_intern("binary")), rb_intern("binary"), 1.0, T, T, T, F, F, F, F);
-    do_field_prop_test(tc, frt_fis_get_field(fis, rb_intern("array")), rb_intern("array"), 1.0, T, T, T, F, F, F, F);
+    do_field_prop_test(tc, frt_fis_get_field(fis, rb_intern("binary")), rb_intern("binary"), 1.0, T, F, T, T, F, F, F, F);
+    do_field_prop_test(tc, frt_fis_get_field(fis, rb_intern("array")), rb_intern("array"), 1.0, T, F, T, T, F, F, F, F);
 
     fr = frt_fr_open(store, "_0", fis);
     doc = frt_fr_get_doc(fr, 0);
@@ -401,12 +413,12 @@ static void test_fields_rw_multi(TestCase *tc, void *data)
     frt_fw_close(fw);
 
     Aiequal(106, fis->size);
-    do_field_prop_test(tc, frt_fis_get_field(fis, rb_intern("binary")), rb_intern("binary"), 1.0, T, T, T, F, F, F, F);
-    do_field_prop_test(tc, frt_fis_get_field(fis, rb_intern("array")), rb_intern("array"), 1.0, T, T, T, F, F, F, F);
+    do_field_prop_test(tc, frt_fis_get_field(fis, rb_intern("binary")), rb_intern("binary"), 1.0, T, F, T, T, F, F, F, F);
+    do_field_prop_test(tc, frt_fis_get_field(fis, rb_intern("array")), rb_intern("array"), 1.0, T, F, T, T, F, F, F, F);
     for (i = 0; i < 100; i++) {
         char buf[100];
         sprintf(buf, "<<%d>>", i);
-        do_field_prop_test(tc, frt_fis_get_field(fis, rb_intern(buf)), rb_intern(buf), 1.0, T, T, T, F, F, F, F);
+        do_field_prop_test(tc, frt_fis_get_field(fis, rb_intern(buf)), rb_intern(buf), 1.0, T, F, T, T, F, F, F, F);
     }
 
     fr = frt_fr_open(store, "_as3", fis);

data/ext/isomorfeus_ferret_ext/test_index.c

@@ -1365,7 +1365,7 @@ static ReaderTestEnvironment *reader_test_env_new(int type)
                                                   FRT_TERM_VECTOR_NO));
                     } else if (compressed_field == df->name) {
                         frt_fis_add_field(fis, frt_fi_new(compressed_field,
-                                                  FRT_STORE_YES,
+                                                  FRT_STORE_COMPRESS,
                                                   FRT_INDEX_YES,
                                                   FRT_TERM_VECTOR_NO));
                     }
@@ -1428,6 +1428,8 @@ static void write_ir_test_docs(FrtStore *store)
                               FRT_TERM_VECTOR_NO));
     frt_fis_add_field(fis, frt_fi_new(text, FRT_STORE_NO, FRT_INDEX_YES,
                               FRT_TERM_VECTOR_NO));
+    frt_fis_add_field(fis, frt_fi_new(compressed_field, FRT_STORE_COMPRESS, FRT_INDEX_YES,
+                              FRT_TERM_VECTOR_NO));
     frt_index_create(store, fis);
     frt_fis_deref(fis);
     config.max_buffered_docs = 5;
@@ -2123,6 +2125,7 @@ static void test_ir_multivalue_fields(TestCase *tc, void *data)
 
     fi = frt_fis_get_field(iw->fis, tag);
     Aiequal(true, fi_is_stored(fi));
+    Aiequal(false, fi_is_compressed(fi));
     Aiequal(true, fi_is_indexed(fi));
     Aiequal(true, fi_is_tokenized(fi));
     Aiequal(true, fi_has_norms(fi));

data/lib/isomorfeus/ferret/version.rb

@@ -1,5 +1,5 @@
 module Isomorfeus
   module Ferret
-    VERSION = '0.12.6'
+    VERSION = '0.12.7'
   end
 end