isomorfeus-ferret 0.17.2 → 0.17.3

data/ext/isomorfeus_ferret_ext/frt_index.h

@@ -2,6 +2,7 @@
 #define FRT_INDEX_H
 
 #include "frt_global.h"
+#include "frt_field_infos.h"
 #include "frt_document.h"
 #include "frt_analysis.h"
 #include "frt_hash.h"
@@ -56,84 +57,6 @@ extern FrtCacheObject *frt_co_create(FrtHash *ref_tab1, FrtHash *ref_tab2,
             void *ref1, void *ref2, frt_free_ft destroy, void *obj);
 extern FrtHash *frt_co_hash_create();
 
-/****************************************************************************
- *
- * FrtFieldInfo
- *
- ****************************************************************************/
-
-#define FRT_FI_DEFAULTS_BM  FRT_FI_IS_STORED_BM | FRT_FI_IS_INDEXED_BM | FRT_FI_IS_TOKENIZED_BM | FRT_FI_STORE_TERM_VECTOR_BM | FRT_FI_STORE_POSITIONS_BM | FRT_FI_STORE_OFFSETS_BM
-#define FRT_FI_IS_STORED_BM          0x001
-#define FRT_FI_IS_COMPRESSED_BM      0x002
-#define FRT_FI_IS_INDEXED_BM         0x004
-#define FRT_FI_IS_TOKENIZED_BM       0x008
-#define FRT_FI_OMIT_NORMS_BM         0x010
-#define FRT_FI_STORE_TERM_VECTOR_BM  0x020
-#define FRT_FI_STORE_POSITIONS_BM    0x040
-#define FRT_FI_STORE_OFFSETS_BM      0x080
-#define FRT_FI_COMPRESSION_BROTLI_BM 0x100
-#define FRT_FI_COMPRESSION_BZ2_BM    0x200
-#define FRT_FI_COMPRESSION_LZ4_BM    0x400
-
-typedef struct FrtFieldInfo {
-    ID           name;
-    float        boost;
-    unsigned int bits;
-    int          number;
-    _Atomic unsigned int  ref_cnt;
-    VALUE        rfi;
-} FrtFieldInfo;
-
-extern FrtFieldInfo *frt_fi_alloc();
-extern FrtFieldInfo *frt_fi_init(FrtFieldInfo *fi, ID name, unsigned int bits);
-extern FrtFieldInfo *frt_fi_new(ID name, unsigned int bits);
-extern char *frt_fi_to_s(FrtFieldInfo *fi);
-extern void frt_fi_deref(FrtFieldInfo *fi);
-
-#define bits_is_stored(bits)            ((bits & FRT_FI_IS_STORED_BM) != 0)
-#define bits_is_compressed(bits)        ((bits & FRT_FI_IS_COMPRESSED_BM) != 0)
-#define bits_is_compressed_brotli(bits) ((bits & FRT_FI_COMPRESSION_BROTLI_BM) != 0)
-#define bits_is_compressed_bz2(bits)    ((bits & FRT_FI_COMPRESSION_BZ2_BM) != 0)
-#define bits_is_compressed_lz4(bits)    ((bits & FRT_FI_COMPRESSION_LZ4_BM) != 0)
-#define bits_is_indexed(bits)           ((bits & FRT_FI_IS_INDEXED_BM) != 0)
-#define bits_is_tokenized(bits)         ((bits & FRT_FI_IS_TOKENIZED_BM) != 0)
-#define bits_omit_norms(bits)           ((bits & FRT_FI_OMIT_NORMS_BM) != 0)
-#define bits_store_term_vector(bits)    ((bits & FRT_FI_STORE_TERM_VECTOR_BM) != 0)
-#define bits_store_positions(bits)      ((bits & FRT_FI_STORE_POSITIONS_BM) != 0)
-#define bits_store_offsets(bits)        ((bits & FRT_FI_STORE_OFFSETS_BM) != 0)
-#define bits_has_norms(bits)\
-    ((bits & (FRT_FI_OMIT_NORMS_BM|FRT_FI_IS_INDEXED_BM)) == FRT_FI_IS_INDEXED_BM)
-
-/****************************************************************************
- *
- * FrtFieldInfos
- *
- ****************************************************************************/
-
-#define FIELD_INFOS_INIT_CAPA 4
-/* carry changes over to dummy_fis in test/test_segments.c */
-typedef struct FrtFieldInfos {
-    unsigned int bits;
-    int          size;
-    int          capa;
-    FrtFieldInfo **fields;
-    FrtHash      *field_dict;
-    _Atomic unsigned int ref_cnt;
-    VALUE        rfis;
-} FrtFieldInfos;
-
-FrtFieldInfos *frt_fis_alloc();
-FrtFieldInfos *frt_fis_init(FrtFieldInfos *fis, unsigned int bits);
-FrtFieldInfos *frt_fis_new(unsigned int bits);
-extern FrtFieldInfo *frt_fis_add_field(FrtFieldInfos *fis, FrtFieldInfo *fi);
-extern FrtFieldInfo *frt_fis_get_field(FrtFieldInfos *fis, ID name);
-extern int frt_fis_get_field_num(FrtFieldInfos *fis, ID name);
-extern FrtFieldInfo *frt_fis_get_or_add_field(FrtFieldInfos *fis, ID name);
-extern void frt_fis_write(FrtFieldInfos *fis, FrtOutStream *os);
-extern FrtFieldInfos *frt_fis_read(FrtInStream *is);
-extern char *frt_fis_to_s(FrtFieldInfos *fis);
-extern void frt_fis_deref(FrtFieldInfos *fis);
-
 /****************************************************************************
  *
  * FrtSegmentInfo
@@ -771,28 +694,4 @@ extern int frt_iw_doc_count(FrtIndexWriter *iw);
 extern void frt_iw_commit(FrtIndexWriter *iw);
 extern void frt_iw_add_readers(FrtIndexWriter *iw, FrtIndexReader **readers, const int r_cnt);
 
-/****************************************************************************
- *
- * FrtCompoundWriter
- *
- ****************************************************************************/
-
-#define FRT_CW_INIT_CAPA 16
-typedef struct FrtCWFileEntry {
-    char  *name;
-    frt_off_t dir_offset;
-    frt_off_t data_offset;
-} FrtCWFileEntry;
-
-typedef struct FrtCompoundWriter {
-    FrtStore       *store;
-    const char     *name;
-    FrtHashSet     *ids;
-    FrtCWFileEntry *file_entries;
-} FrtCompoundWriter;
-
-extern FrtCompoundWriter *frt_open_cw(FrtStore *store, char *name);
-extern void frt_cw_add_file(FrtCompoundWriter *cw, char *id);
-extern void frt_cw_close(FrtCompoundWriter *cw, FrtDeleter *dlr);
-
 #endif

data/ext/isomorfeus_ferret_ext/frt_lang.c

@@ -8,8 +8,7 @@
 #include "frt_global.h"
 
 /* emalloc: malloc and report if error */
-void *frt_emalloc(size_t size)
-{
+void *frt_emalloc(size_t size) {
     void *p = malloc(size);
 
     if (p == NULL) {
@@ -20,8 +19,7 @@ void *frt_emalloc(size_t size)
 }
 
 /* frt_ecalloc: malloc, zeroset and report if error */
-void *frt_ecalloc(size_t size)
-{
+void *frt_ecalloc(size_t size) {
     void *p = calloc(1, size);
 
     if (p == NULL) {
@@ -32,8 +30,7 @@ void *frt_ecalloc(size_t size)
 }
 
 /* frt_erealloc: realloc and report if error */
-void *frt_erealloc(void *ptr, size_t size)
-{
+void *frt_erealloc(void *ptr, size_t size) {
     void *p = realloc(ptr, size);
 
     if (p == NULL) {
@@ -47,8 +44,7 @@ void frt_micro_sleep(const int micro_seconds) {
     rb_thread_wait_for(rb_time_interval(rb_float_new((double)micro_seconds/1000000.0)));
 }
 
-/* void frt_micro_sleep(const int micro_seconds)
-{
+/* void frt_micro_sleep(const int micro_seconds) {
 #if (defined POSH_OS_WIN32 || defined POSH_OS_WIN64) && !defined __MINGW32__
     Sleep(micro_seconds / 1000);
 #else
@@ -81,8 +77,7 @@ void FRT_VEXIT(const char *err_type, const char *fmt, va_list args)
     fprintf(EXCEPTION_STREAM, "\n");
     if (frt_x_abort_on_exception) {
         exit(2);                 /* conventional value for failed execution */
-    }
-    else {
+    } else {
         frt_x_has_aborted = true;
     }
 }

data/ext/isomorfeus_ferret_ext/frt_lazy_doc.c

@@ -1,10 +1,10 @@
 #include "frt_lazy_doc.h"
 
-FrtLazyDoc *frt_lazy_doc_new(int size, FrtInStream *fdt_in) {
+FrtLazyDoc *frt_lazy_doc_new(int field_count, FrtInStream *fdt_in) {
     FrtLazyDoc *self = FRT_ALLOC(FrtLazyDoc);
     self->field_dictionary = frt_h_new_ptr((frt_free_ft)&frt_lazy_df_destroy);
-    self->size = size;
-    self->fields = FRT_ALLOC_AND_ZERO_N(FrtLazyDocField *, size);
+    self->field_count = field_count;
+    self->fields = FRT_ALLOC_AND_ZERO_N(FrtLazyDocField *, field_count);
     self->fields_in = frt_is_clone(fdt_in);
     self->loaded = false;
     return self;

data/ext/isomorfeus_ferret_ext/frt_lazy_doc.h

@@ -5,7 +5,7 @@
 
 typedef struct FrtLazyDoc {
     FrtHash         *field_dictionary;
-    int             size;
+    int             field_count;
     FrtLazyDocField **fields;
     FrtInStream     *fields_in;
     bool            loaded;

data/ext/isomorfeus_ferret_ext/frt_lazy_doc_field.c

@@ -2,12 +2,12 @@
 #include "frt_lazy_doc_field.h"
 #include "frt_lazy_doc.h"
 
-FrtLazyDocField *frt_lazy_df_new(ID name, const int size, FrtCompressionType compression) {
+FrtLazyDocField *frt_lazy_df_new(ID name, const int size, FrtCompressionType compression_type) {
     FrtLazyDocField *self = FRT_ALLOC(FrtLazyDocField);
     self->name = name;
     self->size = size;
     self->data = FRT_ALLOC_AND_ZERO_N(FrtLazyDocFieldData, size);
-    self->compression = compression;
+    self->compression_type = compression_type;
     self->decompressed = false;
     self->loaded = false;
     return self;
@@ -17,26 +17,27 @@ void frt_lazy_df_destroy(FrtLazyDocField *self) {
     int i;
     for (i = self->size - 1; i >= 0; i--) {
         if (self->data[i].text) {
-            free(self->data[i].text);
+            free((void *)self->data[i].text);
          }
     }
     free(self->data);
     free(self);
 }
 
-char *frt_lazy_df_get_data(FrtLazyDocField *self, int i) {
-    char *text = NULL;
+const char *frt_lazy_df_get_data(FrtLazyDocField *self, int i) {
+    const char *text = NULL;
     if (i < self->size && i >= 0) {
         text = self->data[i].text;
         if (NULL == text) {
-            const int read_len = self->data[i].length + 1;
+            const int read_len = self->data[i].length;
             frt_is_seek(self->doc->fields_in, self->data[i].start);
-            if (self->data[i].compression != FRT_COMPRESSION_NONE) {
-                self->data[i].text = text = frt_is_read_compressed_bytes(self->doc->fields_in, read_len, &(self->data[i].length), self->data[i].compression);
+            if (self->data[i].compression_type != FRT_COMPRESSION_NONE) {
+                self->data[i].text = text = frt_is_read_compressed_bytes(self->doc->fields_in, read_len, &(self->data[i].length), self->data[i].compression_type);
             } else {
-                self->data[i].text = text = FRT_ALLOC_N(char, read_len);
-                frt_is_read_bytes(self->doc->fields_in, (frt_uchar *)text, read_len);
-                text[read_len - 1] = '\0';
+                char *d = FRT_ALLOC_N(char, read_len + 1);
+                frt_is_read_bytes(self->doc->fields_in, (frt_uchar *)d, read_len);
+                d[read_len] = '\0';
+                self->data[i].text = text = d;
             }
             self->loaded = true;
         }
@@ -46,14 +47,13 @@ char *frt_lazy_df_get_data(FrtLazyDocField *self, int i) {
 }
 
 void frt_lazy_df_get_bytes(FrtLazyDocField *self, char *buf, int start, int len) {
-    if (self->compression != FRT_COMPRESSION_NONE && !self->decompressed) {
+    if (self->compression_type != FRT_COMPRESSION_NONE && !self->decompressed) {
         int i;
         self->len = 0;
         for (i = self->size-1; i >= 0; i--) {
             (void)frt_lazy_df_get_data(self, i);
-            self->len += self->data[i].length + 1;
+            self->len += self->data[i].length;
         }
-        self->len--; /* each field separated by ' ' but no need to add to end */
         self->decompressed = true;
     }
     if (start < 0 || start >= self->len) {
@@ -67,7 +67,7 @@ void frt_lazy_df_get_bytes(FrtLazyDocField *self, char *buf, int start, int len)
         FRT_RAISE(FRT_IO_ERROR, "Tried to read past end of field. Field is only %d "
               "bytes long but tried to read to %d", self->len, start + len);
     }
-    if (self->compression != FRT_COMPRESSION_NONE) {
+    if (self->compression_type != FRT_COMPRESSION_NONE) {
         int cur_start = 0, buf_start = 0, cur_end, i, copy_start, copy_len;
         for (i = 0; i < self->size; i++) {
             cur_end = cur_start + self->data[i].length;
@@ -77,21 +77,14 @@ void frt_lazy_df_get_bytes(FrtLazyDocField *self, char *buf, int start, int len)
                 if (copy_len >= len) {
                     copy_len = len;
                     len = 0;
-                }
-                else {
+                } else {
                     len -= copy_len;
                 }
-                memcpy(buf + buf_start,
-                       self->data[i].text + copy_start,
-                       copy_len);
+                memcpy(buf + buf_start, self->data[i].text + copy_start, copy_len);
                 buf_start += copy_len;
-                if (len > 0) {
-                    buf[buf_start++] = ' ';
-                    len--;
-                }
                 if (len == 0) break;
             }
-            cur_start = cur_end + 1;
+            cur_start = cur_end;
         }
     } else {
         frt_is_seek(self->doc->fields_in, self->data[0].start + start);

data/ext/isomorfeus_ferret_ext/frt_lazy_doc_field.h

@@ -9,8 +9,8 @@ typedef struct FrtLazyDocFieldData {
     frt_off_t          start;
     int                length;
     rb_encoding        *encoding;
-    FrtCompressionType compression; /* as stored */
-    char               *text;
+    FrtCompressionType compression_type; /* as stored */
+    const char         *text;
 } FrtLazyDocFieldData;
 
 typedef struct FrtLazyDoc FrtLazyDoc;
@@ -20,13 +20,13 @@ typedef struct FrtLazyDocField {
     FrtLazyDoc          *doc;
     int                 size; /* number of data elements */
     int                 len;  /* length of data elements concatenated */
-    FrtCompressionType  compression; /* as configured */
+    FrtCompressionType  compression_type; /* as configured */
     bool                decompressed;
     bool                loaded;
 } FrtLazyDocField;
 
-extern FrtLazyDocField *frt_lazy_df_new(ID name, const int size, FrtCompressionType compression);
-extern char *frt_lazy_df_get_data(FrtLazyDocField *self, int i);
+extern FrtLazyDocField *frt_lazy_df_new(ID name, const int size, FrtCompressionType compression_type);
+extern const char *frt_lazy_df_get_data(FrtLazyDocField *self, int i);
 extern void frt_lazy_df_get_bytes(FrtLazyDocField *self, char *buf, int start, int len);
 extern void frt_lazy_df_destroy(FrtLazyDocField *self);