isomorfeus-ferret 0.17.2 → 0.17.3

data/ext/isomorfeus_ferret_ext/frt_field_info.c

@@ -0,0 +1,69 @@
+#include "frt_field_info.h"
+
+FrtFieldInfo *frt_fi_alloc(void) {
+    return FRT_ALLOC(FrtFieldInfo);
+}
+
+FrtFieldInfo *frt_fi_init(FrtFieldInfo *fi, ID name, unsigned int bits) {
+    assert(NULL != name);
+    bits_check(bits);
+    fi->name = name;
+    fi->boost = 1.0f;
+    fi->bits = bits;
+    fi->number = 0;
+    fi->ref_cnt = 1;
+    fi->rfi = Qnil;
+    return fi;
+}
+
+FrtFieldInfo *frt_fi_new(ID name, unsigned int bits) {
+    FrtFieldInfo *fi = frt_fi_alloc();
+    return frt_fi_init(fi, name, bits);
+}
+
+void frt_fi_deref(FrtFieldInfo *fi) {
+    if (FRT_DEREF(fi) == 0) free(fi);
+}
+
+void bits_check(unsigned int bits) {
+    if (!bits_is_indexed(bits) && bits_store_term_vector(bits)) {
+        FRT_RAISE(FRT_ARG_ERROR, "You can't store the term vectors of an unindexed field.");
+    }
+    if (bits_is_compressed(bits) && !bits_is_stored(bits)) {
+        FRT_RAISE(FRT_ARG_ERROR, "Field must be stored for compression to be useful.");
+    }
+}
+
+FrtCompressionType bits_get_compression_type(unsigned int bits) {
+    if (bits_is_compressed_brotli(bits)) {
+        return FRT_COMPRESSION_BROTLI;
+    } else if (bits_is_compressed_bz2(bits)) {
+        return FRT_COMPRESSION_BZ2;
+    } else if (bits_is_compressed_lz4(bits)) {
+        return FRT_COMPRESSION_LZ4;
+    } else {
+        return FRT_COMPRESSION_NONE;
+    }
+}
+
+char *frt_fi_to_s(FrtFieldInfo *fi) {
+    const char *fi_name = rb_id2name(fi->name);
+    char *str = FRT_ALLOC_N(char, strlen(fi_name) + 200);
+    char *s = str;
+    s += sprintf(str, "[\"%s\":(%s%s%s%s%s%s%s%s", fi_name,
+                 bits_is_stored(fi->bits) ? "is_stored, " : "",
+                 bits_is_compressed(fi->bits) ? "is_compressed, " : "",
+                 bits_is_indexed(fi->bits) ? "is_indexed, " : "",
+                 bits_is_tokenized(fi->bits) ? "is_tokenized, " : "",
+                 bits_omit_norms(fi->bits) ? "omit_norms, " : "",
+                 bits_store_term_vector(fi->bits) ? "store_term_vector, " : "",
+                 bits_store_positions(fi->bits) ? "store_positions, " : "",
+                 bits_store_offsets(fi->bits) ? "store_offsets, " : "");
+    s -= 2;
+    if (*s != ',') {
+        s += 2;
+    }
+
+    sprintf(s, ")]");
+    return str;
+}

data/ext/isomorfeus_ferret_ext/frt_field_info.h

@@ -0,0 +1,49 @@
+#ifndef FRT_FIELD_INFO_H
+#define FRT_FIELD_INFO_H
+
+#include "frt_global.h"
+#include <ruby.h>
+
+#define FRT_FI_DEFAULTS_BM  FRT_FI_IS_STORED_BM | FRT_FI_IS_INDEXED_BM | FRT_FI_IS_TOKENIZED_BM | FRT_FI_STORE_TERM_VECTOR_BM | FRT_FI_STORE_POSITIONS_BM | FRT_FI_STORE_OFFSETS_BM
+#define FRT_FI_IS_STORED_BM          0x001
+#define FRT_FI_IS_INDEXED_BM         0x002
+#define FRT_FI_IS_TOKENIZED_BM       0x004
+#define FRT_FI_OMIT_NORMS_BM         0x008
+#define FRT_FI_STORE_TERM_VECTOR_BM  0x010
+#define FRT_FI_STORE_POSITIONS_BM    0x020
+#define FRT_FI_STORE_OFFSETS_BM      0x040
+#define FRT_FI_COMPRESSION_BROTLI_BM 0x080
+#define FRT_FI_COMPRESSION_BZ2_BM    0x100
+#define FRT_FI_COMPRESSION_LZ4_BM    0x200
+
+typedef struct FrtFieldInfo {
+    ID           name;
+    float        boost;
+    unsigned int bits;
+    int          number;
+    _Atomic unsigned int ref_cnt;
+    VALUE        rfi;
+} FrtFieldInfo;
+
+extern FrtFieldInfo *frt_fi_alloc();
+extern FrtFieldInfo *frt_fi_init(FrtFieldInfo *fi, ID name, unsigned int bits);
+extern FrtFieldInfo *frt_fi_new(ID name, unsigned int bits);
+extern char *frt_fi_to_s(FrtFieldInfo *fi);
+extern void frt_fi_deref(FrtFieldInfo *fi);
+
+extern void bits_check(unsigned int bits);
+extern FrtCompressionType bits_get_compression_type(unsigned int bits);
+#define bits_is_stored(bits)            ((bits & FRT_FI_IS_STORED_BM) != 0)
+#define bits_is_indexed(bits)           ((bits & FRT_FI_IS_INDEXED_BM) != 0)
+#define bits_is_tokenized(bits)         ((bits & FRT_FI_IS_TOKENIZED_BM) != 0)
+#define bits_omit_norms(bits)           ((bits & FRT_FI_OMIT_NORMS_BM) != 0)
+#define bits_store_term_vector(bits)    ((bits & FRT_FI_STORE_TERM_VECTOR_BM) != 0)
+#define bits_store_positions(bits)      ((bits & FRT_FI_STORE_POSITIONS_BM) != 0)
+#define bits_store_offsets(bits)        ((bits & FRT_FI_STORE_OFFSETS_BM) != 0)
+#define bits_has_norms(bits)            ((bits & (FRT_FI_OMIT_NORMS_BM|FRT_FI_IS_INDEXED_BM)) == FRT_FI_IS_INDEXED_BM)
+#define bits_is_compressed_brotli(bits) ((bits & FRT_FI_COMPRESSION_BROTLI_BM) != 0)
+#define bits_is_compressed_bz2(bits)    ((bits & FRT_FI_COMPRESSION_BZ2_BM) != 0)
+#define bits_is_compressed_lz4(bits)    ((bits & FRT_FI_COMPRESSION_LZ4_BM) != 0)
+#define bits_is_compressed(bits)        (bits_is_compressed_brotli(bits) || bits_is_compressed_bz2(bits) || bits_is_compressed_lz4(bits))
+
+#endif

data/ext/isomorfeus_ferret_ext/frt_field_infos.c

@@ -0,0 +1,196 @@
+#include "frt_field_infos.h"
+#include "frt_except.h"
+
+FrtFieldInfos *frt_fis_alloc(void) {
+    return FRT_ALLOC(FrtFieldInfos);
+}
+
+FrtFieldInfos *frt_fis_init(FrtFieldInfos *fis, unsigned int bits) {
+    bits_check(bits);
+    fis->field_dict = frt_h_new_ptr((frt_free_ft)&frt_fi_deref);
+    fis->size = 0;
+    fis->capa = FIELD_INFOS_INIT_CAPA;
+    fis->fields = FRT_ALLOC_N(FrtFieldInfo *, fis->capa);
+    fis->bits = bits;
+    fis->ref_cnt = 1;
+    fis->rfis = Qnil;
+    return fis;
+}
+
+FrtFieldInfos *frt_fis_new(unsigned int bits) {
+    FrtFieldInfos *fis = frt_fis_alloc();
+    return frt_fis_init(fis, bits);
+}
+
+FrtFieldInfo *frt_fis_add_field(FrtFieldInfos *fis, FrtFieldInfo *fi) {
+    if (fis->size == fis->capa) {
+        fis->capa <<= 1;
+        FRT_REALLOC_N(fis->fields, FrtFieldInfo *, fis->capa);
+    }
+    if (!frt_h_set_safe(fis->field_dict, (void *)fi->name, fi)) {
+        FRT_RAISE(FRT_ARG_ERROR, "Field :%s already exists", rb_id2name(fi->name));
+    }
+    FRT_REF(fi);
+    fi->number = fis->size;
+    fis->fields[fis->size] = fi;
+    fis->size++;
+    return fi;
+}
+
+FrtFieldInfo *frt_fis_get_field(FrtFieldInfos *fis, ID name) {
+    return (FrtFieldInfo *)frt_h_get(fis->field_dict, (void *)name);
+}
+
+int frt_fis_get_field_num(FrtFieldInfos *fis, ID name) {
+    FrtFieldInfo *fi = (FrtFieldInfo *)frt_h_get(fis->field_dict, (void *)name);
+    if (fi) { return fi->number; }
+    else { return -1; }
+}
+
+FrtFieldInfo *frt_fis_get_or_add_field(FrtFieldInfos *fis, ID name) {
+    FrtFieldInfo *fi = (FrtFieldInfo *)frt_h_get(fis->field_dict, (void *)name);
+    if (!fi) {
+        fi = (FrtFieldInfo*)frt_fi_new(name, fis->bits);
+        frt_fis_add_field(fis, fi);
+    }
+    return fi;
+}
+
+bool frt_fis_has_vectors(FrtFieldInfos *fis) {
+    int i;
+    const int fis_size = fis->size;
+
+    for (i = 0; i < fis_size; i++) {
+        if (bits_store_term_vector(fis->fields[i]->bits)) {
+            return true;
+        }
+    }
+    return false;
+}
+
+FrtFieldInfos *frt_fis_read(FrtInStream *is) {
+    FrtFieldInfos *volatile fis = NULL;
+    char *field_name;
+    FRT_TRY
+        do {
+            volatile int i;
+            union { frt_u32 i; float f; } tmp;
+            FrtFieldInfo *volatile fi;
+            fis = frt_fis_new(frt_is_read_vint(is));
+            for (i = frt_is_read_vint(is); i > 0; i--) {
+                fi = FRT_ALLOC_AND_ZERO(FrtFieldInfo);
+                FRT_TRY
+                    field_name = frt_is_read_string_safe(is);
+                    fi->name = rb_intern(field_name);
+                    free(field_name);
+                    tmp.i = frt_is_read_u32(is);
+                    fi->boost = tmp.f;
+                    fi->bits = frt_is_read_vint(is);
+                FRT_XCATCHALL
+                    free(fi);
+                FRT_XENDTRY
+                frt_fis_add_field(fis, fi);
+                fi->ref_cnt = 1;
+            }
+        } while (0);
+    FRT_XCATCHALL
+        frt_fis_deref(fis);
+    FRT_XENDTRY
+    return fis;
+}
+
+void frt_fis_write(FrtFieldInfos *fis, FrtOutStream *os) {
+    int i;
+    union { frt_u32 i; float f; } tmp;
+    FrtFieldInfo *fi;
+    const int fis_size = fis->size;
+
+    frt_os_write_vint(os, fis->bits);
+    frt_os_write_vint(os, fis->size);
+
+    for (i = 0; i < fis_size; i++) {
+        fi = fis->fields[i];
+
+        frt_os_write_string(os, rb_id2name(fi->name));
+        tmp.f = fi->boost;
+        frt_os_write_u32(os, tmp.i);
+        frt_os_write_vint(os, fi->bits);
+    }
+}
+
+static const char *store_str[] = {
+    ":no",
+    ":yes",
+    ":compressed"
+};
+
+static const char *fi_store_str(FrtFieldInfo *fi) {
+    return store_str[bits_is_compressed(fi->bits) ? 2 : fi->bits & 0x1];
+}
+
+static const char *index_str[] = {
+    ":no",
+    ":untokenized",
+    "",
+    ":yes",
+    "",
+    ":untokenized_omit_norms",
+    "",
+    ":omit_norms"
+};
+
+static const char *fi_index_str(FrtFieldInfo *fi) {
+    return index_str[(fi->bits >> 1) & 0x7];
+}
+
+static const char *term_vector_str[] = {
+    ":no",
+    ":yes",
+    "",
+    ":with_positions",
+    "",
+    ":with_offsets",
+    "",
+    ":with_positions_offsets"
+};
+
+static const char *fi_term_vector_str(FrtFieldInfo *fi) {
+    return term_vector_str[(fi->bits >> 4) & 0x7];
+}
+
+char *frt_fis_to_s(FrtFieldInfos *fis) {
+    int i, pos, capa = 200 + fis->size * 120;
+    char *buf = FRT_ALLOC_N(char, capa);
+    FrtFieldInfo *fi;
+    const int fis_size = fis->size;
+
+    pos = sprintf(buf,
+                  "default:\n"
+                  "  store: %s\n"
+                  "  index: %s\n"
+                  "  term_vector: %s\n"
+                  "fields:\n",
+                  store_str[bits_is_compressed(fis->bits) ? 2 : fis->bits & 0x1],
+                  index_str[(fis->bits >> 1) & 0x7],
+                  term_vector_str[(fis->bits >> 4) & 0x7]);
+    for (i = 0; i < fis_size; i++) {
+        fi = fis->fields[i];
+        pos += sprintf(buf + pos,
+                       "  %s:\n"
+                       "    boost: %f\n"
+                       "    store: %s\n"
+                       "    index: %s\n"
+                       "    term_vector: %s\n",
+                       rb_id2name(fi->name), fi->boost, fi_store_str(fi),
+                       fi_index_str(fi), fi_term_vector_str(fi));
+    }
+    return buf;
+}
+
+void frt_fis_deref(FrtFieldInfos *fis) {
+    if (FRT_DEREF(fis) == 0) {
+        frt_h_destroy(fis->field_dict);
+        free(fis->fields);
+        free(fis);
+    }
+}

data/ext/isomorfeus_ferret_ext/frt_field_infos.h

@@ -0,0 +1,35 @@
+#ifndef FRT_FIELD_INFOS_H
+#define FRT_FIELD_INFOS_H
+
+#include "frt_field_info.h"
+#include "frt_hash.h"
+#include "frt_in_stream.h"
+#include "frt_out_stream.h"
+
+#define FIELD_INFOS_INIT_CAPA 4
+
+/* carry changes over to dummy_fis in test/test_segments.c */
+typedef struct FrtFieldInfos {
+    unsigned int bits;
+    int          size;
+    int          capa;
+    FrtFieldInfo **fields;
+    FrtHash      *field_dict;
+    _Atomic unsigned int ref_cnt;
+    VALUE        rfis;
+} FrtFieldInfos;
+
+FrtFieldInfos *frt_fis_alloc();
+FrtFieldInfos *frt_fis_init(FrtFieldInfos *fis, unsigned int bits);
+FrtFieldInfos *frt_fis_new(unsigned int bits);
+extern FrtFieldInfo *frt_fis_add_field(FrtFieldInfos *fis, FrtFieldInfo *fi);
+extern FrtFieldInfo *frt_fis_get_field(FrtFieldInfos *fis, ID name);
+extern int frt_fis_get_field_num(FrtFieldInfos *fis, ID name);
+extern FrtFieldInfo *frt_fis_get_or_add_field(FrtFieldInfos *fis, ID name);
+extern bool frt_fis_has_vectors(FrtFieldInfos *fis);
+extern void frt_fis_write(FrtFieldInfos *fis, FrtOutStream *os);
+extern FrtFieldInfos *frt_fis_read(FrtInStream *is);
+extern char *frt_fis_to_s(FrtFieldInfos *fis);
+extern void frt_fis_deref(FrtFieldInfos *fis);
+
+#endif

data/ext/isomorfeus_ferret_ext/frt_global.c

@@ -44,8 +44,7 @@ int frt_icmp(const void *p1, const void *p2) {
 
     if (i1 > i2) {
         return 1;
-    }
-    else if (i1 < i2) {
+    } else if (i1 < i2) {
         return -1;
     }
     return 0;
@@ -110,6 +109,14 @@ char *frt_estrdup(const char *s) {
     return t;
 }
 
+/* frt_estrndup: duplicate a string with length given, report if error */
+char *frt_estrndup(const char *s, int len) {
+    char *t = FRT_ALLOC_N(char, len + 1);
+    strncpy(t, s, len);
+    t[len] = '\0';
+    return t;
+}
+
 /* Pretty print a float to the buffer. The buffer should have at least 32
  * bytes available.
  */
@@ -248,8 +255,7 @@ void frt_register_for_cleanup(void *p, frt_free_ft free_func) {
     if (free_mes_capa == 0) {
         free_mes_capa = 16;
         free_mes = FRT_ALLOC_N(FreeMe, free_mes_capa);
-    }
-    else if (free_mes_capa <= free_mes_size) {
+    } else if (free_mes_capa <= free_mes_size) {
         free_mes_capa *= 2;
         FRT_REALLOC_N(free_mes, FreeMe, free_mes_capa);
     }

data/ext/isomorfeus_ferret_ext/frt_global.h

@@ -83,12 +83,15 @@ typedef void (*frt_free_ft)(void *key);
 # define Xj fprintf(stdout,"%s, %d: %s\n", __FILE__, __LINE__, __func__);
 #endif
 
+extern const char *segm_idx_name;
+
 extern unsigned int *frt_imalloc(unsigned int value);
 extern unsigned long *frt_lmalloc(unsigned long value);
 extern frt_u32 *frt_u32malloc(frt_u32 value);
 extern frt_u64 *frt_u64malloc(frt_u64 value);
 
 extern char *frt_estrdup(const char *s);
+extern char *frt_estrndup(const char *s, int len);
 extern char *frt_estrcat(char *str, char *str_cat);
 extern char *frt_epstrdup(const char *fmt, int len, ...);
 
@@ -120,8 +123,7 @@ extern void frt_dummy_free(void *p);
  * Returns the count of leading [MSB] 0 bits in +word+.
  */
 static FRT_ATTR_ALWAYS_INLINE FRT_ATTR_CONST
-int frt_count_leading_zeros(frt_u32 word)
-{
+int frt_count_leading_zeros(frt_u32 word) {
 #ifdef __GNUC__
     if (word)
         return __builtin_clz(word);
@@ -153,8 +155,7 @@ int frt_count_leading_zeros(frt_u32 word)
 }
 
 static FRT_ATTR_ALWAYS_INLINE FRT_ATTR_CONST
-int frt_count_leading_ones(frt_u32 word)
-{
+int frt_count_leading_ones(frt_u32 word) {
     return frt_count_leading_zeros(~word);
 }
 
@@ -163,8 +164,7 @@ int frt_count_leading_ones(frt_u32 word)
  */
 
 static FRT_ATTR_ALWAYS_INLINE FRT_ATTR_CONST
-int frt_count_trailing_zeros(frt_u32 word)
-{
+int frt_count_trailing_zeros(frt_u32 word) {
 #ifdef __GNUC__
     if (word)
         return __builtin_ctz(word);
@@ -196,14 +196,12 @@ int frt_count_trailing_zeros(frt_u32 word)
 }
 
 static FRT_ATTR_ALWAYS_INLINE FRT_ATTR_CONST
-int frt_count_trailing_ones(frt_u32 word)
-{
+int frt_count_trailing_ones(frt_u32 word) {
     return frt_count_trailing_zeros(~word);
 }
 
 static FRT_ATTR_ALWAYS_INLINE FRT_ATTR_CONST
-int frt_count_ones(frt_u32 word)
-{
+int frt_count_ones(frt_u32 word) {
 #ifdef __GNUC__
     return __builtin_popcount(word);
 #else
@@ -233,8 +231,7 @@ int frt_count_ones(frt_u32 word)
 }
 
 static FRT_ATTR_ALWAYS_INLINE FRT_ATTR_CONST
-int frt_count_zeros(frt_u32 word)
-{
+int frt_count_zeros(frt_u32 word) {
     return frt_count_ones(~word);
 }
 
@@ -242,8 +239,7 @@ int frt_count_zeros(frt_u32 word)
  * Round up to the next power of 2
  */
 static FRT_ATTR_ALWAYS_INLINE FRT_ATTR_CONST
-int frt_round2(frt_u32 word)
-{
+int frt_round2(frt_u32 word) {
     return 1 << (32 - frt_count_leading_zeros(word));
 }
 
@@ -261,7 +257,7 @@ extern FILE *frt_x_exception_stream;
 
 /**
  * The convenience macro +EXCEPTION_STREAM+ returns stderr when
- * +frt_x_exception_stream+ isn't explicitely set.
+ * +frt_x_exception_stream+ isn't explicitly set.
  */
 #define EXCEPTION 2
 #define EXCEPTION_STREAM (frt_x_exception_stream ? frt_x_exception_stream : stderr)

data/ext/isomorfeus_ferret_ext/frt_hash.c

@@ -30,7 +30,7 @@ unsigned long frt_str_hash(const char *const str) {
 }
 
 unsigned long frt_ptr_hash(const void *const ptr) {
-    return (unsigned long)ptr;
+    return (unsigned long)(uintptr_t)ptr;
 }
 
 int frt_ptr_eq(const void *q1, const void *q2) {
@@ -73,7 +73,7 @@ static FrtHashEntry  *frt_h_resize_lookup(FrtHash *self, register const unsigned
 }
 
 static FrtHashEntry *frt_h_lookup_ptr(FrtHash *self, const void *key) {
-    register const unsigned long hash = (unsigned long)key;
+    register const unsigned long hash = (unsigned long)(uintptr_t)key;
     register unsigned long perturb;
     register int mask = self->mask;
     register FrtHashEntry *he0 = self->table;
@@ -375,21 +375,21 @@ FrtHashKeyStatus frt_h_has_key(FrtHash *self, const void *key) {
 }
 
 void *frt_h_get_int(FrtHash *self, const unsigned long key) {
-    return frt_h_get(self, (const void *)key);
+    return frt_h_get(self, (const void *)(uintptr_t)key);
 }
 
 int frt_h_del_int(FrtHash *self, const unsigned long key) {
-    return frt_h_del(self, (const void *)key);
+    return frt_h_del(self, (const void *)(uintptr_t)key);
 }
 
 void *frt_h_rem_int(FrtHash *self, const unsigned long key) {
-    return frt_h_rem(self, (const void *)key, false);
+    return frt_h_rem(self, (const void *)(uintptr_t)key, false);
 }
 
 FrtHashKeyStatus frt_h_set_int(FrtHash *self, const unsigned long key, void *value) {
     FrtHashKeyStatus ret_val = FRT_HASH_KEY_DOES_NOT_EXIST;
     FrtHashEntry *he;
-    if (!frt_h_set_ext(self, (const void *)key, &he)) {
+    if (!frt_h_set_ext(self, (const void *)(uintptr_t)key, &he)) {
         /* Only free old value if it isn't the new value */
         if (he->value != value) {
             self->free_value_i(he->value);
@@ -404,7 +404,7 @@ FrtHashKeyStatus frt_h_set_int(FrtHash *self, const unsigned long key, void *val
 
 int frt_h_set_safe_int(FrtHash *self, const unsigned long key, void *value) {
     FrtHashEntry *he;
-    if (frt_h_set_ext(self, (const void *)key, &he)) {
+    if (frt_h_set_ext(self, (const void *)(uintptr_t)key, &he)) {
         he->key = (char *)dummy_int_key;
         he->value = value;
         return true;
@@ -413,7 +413,7 @@ int frt_h_set_safe_int(FrtHash *self, const unsigned long key, void *value) {
 }
 
 int frt_h_has_key_int(FrtHash *self, const unsigned long key) {
-    return frt_h_has_key(self, (const void *)key);
+    return frt_h_has_key(self, (const void *)(uintptr_t)key);
 }
 
 void frt_h_each(FrtHash *self, void (*each_kv) (void *key, void *value, void *arg), void *arg) {

data/ext/isomorfeus_ferret_ext/frt_hash.h

@@ -15,8 +15,7 @@
 /**
  * Return values for frt_h_set
  */
-typedef enum
-{
+typedef enum {
     FRT_HASH_KEY_DOES_NOT_EXIST = 0,
     FRT_HASH_KEY_EQUAL = 1,
     FRT_HASH_KEY_SAME = 2