isomorfeus-ferret 0.12.7 → 0.13.0

data/ext/isomorfeus_ferret_ext/frt_filter.c

@@ -8,21 +8,18 @@
  *
  ***************************************************************************/
 
-void frt_filt_destroy_i(FrtFilter *filt)
-{
+void frt_filt_destroy_i(FrtFilter *filt) {
     frt_h_destroy(filt->cache);
     free(filt);
 }
 
-void frt_filt_deref(FrtFilter *filt)
-{
+void frt_filt_deref(FrtFilter *filt) {
     if (--(filt->ref_cnt) == 0) {
         filt->destroy_i(filt);
     }
 }
 
-FrtBitVector *frt_filt_get_bv(FrtFilter *filt, FrtIndexReader *ir)
-{
+FrtBitVector *frt_filt_get_bv(FrtFilter *filt, FrtIndexReader *ir) {
     FrtCacheObject *co = (FrtCacheObject *)frt_h_get(filt->cache, ir);
 
     if (!co) {
@@ -37,26 +34,22 @@ FrtBitVector *frt_filt_get_bv(FrtFilter *filt, FrtIndexReader *ir)
     return (FrtBitVector *)co->obj;
 }
 
-static char *filt_to_s_i(FrtFilter *filt)
-{
+static char *filt_to_s_i(FrtFilter *filt) {
     return frt_estrdup(rb_id2name(filt->name));
 }
 
-static unsigned long long frt_filt_hash_default(FrtFilter *filt)
-{
+static unsigned long long frt_filt_hash_default(FrtFilter *filt) {
     (void)filt;
     return 0;
 }
 
-static int frt_filt_eq_default(FrtFilter *filt, FrtFilter *o)
-{
+static int frt_filt_eq_default(FrtFilter *filt, FrtFilter *o) {
     (void)filt; (void)o;
     return false;
 }
 
-FrtFilter *frt_filt_create(size_t size, FrtSymbol name)
-{
-    FrtFilter *filt    = (FrtFilter *)frt_emalloc(size);
+FrtFilter *frt_filt_create(size_t size, ID name) {
+    FrtFilter *filt = (FrtFilter *)frt_emalloc(size);
     filt->cache     = frt_co_hash_create();
     filt->name      = name;
     filt->to_s      = &filt_to_s_i;
@@ -64,46 +57,35 @@ FrtFilter *frt_filt_create(size_t size, FrtSymbol name)
     filt->eq        = &frt_filt_eq_default;
     filt->destroy_i = &frt_filt_destroy_i;
     filt->ref_cnt   = 1;
+    filt->rfilter   = Qnil;
     return filt;
 }
 
-unsigned long long frt_filt_hash(FrtFilter *filt)
-{
+unsigned long long frt_filt_hash(FrtFilter *filt) {
     return frt_str_hash(rb_id2name(filt->name)) ^ filt->hash(filt);
 }
 
-int frt_filt_eq(FrtFilter *filt, FrtFilter *o)
-{
-    return ((filt == o)
-            || ((filt->name == o->name)
-                && (filt->eq == o->eq)
-                && (filt->eq(filt, o))));
+int frt_filt_eq(FrtFilter *filt, FrtFilter *o) {
+    return ((filt == o) || ((filt->name == o->name) && (filt->eq == o->eq) && (filt->eq(filt, o))));
 }
 
 /***************************************************************************
  *
- * QueryFilter
+ * FrtQueryFilter
  *
  ***************************************************************************/
 
-#define QF(filt) ((QueryFilter *)(filt))
-typedef struct QueryFilter
-{
-    FrtFilter super;
-    FrtQuery *query;
-} QueryFilter;
+#define QF(filt) ((FrtQueryFilter *)(filt))
 
-static char *qfilt_to_s(FrtFilter *filt)
-{
+static char *qfilt_to_s(FrtFilter *filt) {
     FrtQuery *query = QF(filt)->query;
-    char *query_str = query->to_s(query, (FrtSymbol)NULL);
+    char *query_str = query->to_s(query, (ID)NULL);
     char *filter_str = frt_strfmt("QueryFilter< %s >", query_str);
     free(query_str);
     return filter_str;
 }
 
-static FrtBitVector *qfilt_get_bv_i(FrtFilter *filt, FrtIndexReader *ir)
-{
+static FrtBitVector *qfilt_get_bv_i(FrtFilter *filt, FrtIndexReader *ir) {
     FrtBitVector *bv = frt_bv_new_capa(ir->max_doc(ir));
     FrtSearcher *sea = frt_isea_new(ir);
     FrtWeight *weight = frt_q_weight(QF(filt)->query, sea);
@@ -119,27 +101,25 @@ static FrtBitVector *qfilt_get_bv_i(FrtFilter *filt, FrtIndexReader *ir)
     return bv;
 }
 
-static unsigned long long qfilt_hash(FrtFilter *filt)
-{
+static unsigned long long qfilt_hash(FrtFilter *filt) {
     return frt_q_hash(QF(filt)->query);
 }
 
-static int qfilt_eq(FrtFilter *filt, FrtFilter *o)
-{
+static int qfilt_eq(FrtFilter *filt, FrtFilter *o) {
     return frt_q_eq(QF(filt)->query, QF(o)->query);
 }
 
-static void qfilt_destroy_i(FrtFilter *filt)
-{
+static void qfilt_destroy_i(FrtFilter *filt) {
     FrtQuery *query = QF(filt)->query;
     frt_q_deref(query);
     frt_filt_destroy_i(filt);
 }
 
-FrtFilter *frt_qfilt_new_nr(FrtQuery *query)
-{
-    FrtFilter *filt = filt_new(QueryFilter);
+FrtFilter *frt_qfilt_alloc(void) {
+    return filt_new(FrtQueryFilter);
+}
 
+FrtFilter *frt_qfilt_init_nr(FrtFilter *filt, FrtQuery *query) {
     QF(filt)->query = query;
 
     filt->get_bv_i  = &qfilt_get_bv_i;
@@ -150,8 +130,17 @@ FrtFilter *frt_qfilt_new_nr(FrtQuery *query)
     return filt;
 }
 
-FrtFilter *frt_qfilt_new(FrtQuery *query)
-{
+FrtFilter *frt_qfilt_new_nr(FrtQuery *query) {
+    FrtFilter *filt = frt_qfilt_alloc();
+    return frt_qfilt_init_nr(filt, query);
+}
+
+FrtFilter *frt_qfilt_init(FrtFilter *filt, FrtQuery *query) {
+    FRT_REF(query);
+    return frt_qfilt_init_nr(filt, query);
+}
+
+FrtFilter *frt_qfilt_new(FrtQuery *query) {
     FRT_REF(query);
     return frt_qfilt_new_nr(query);
 }

data/ext/isomorfeus_ferret_ext/frt_fs_store.c

@@ -405,6 +405,7 @@ static FrtLock *fs_open_lock_i(FrtStore *store, const char *lockname)
     lock->obtain = &fs_lock_obtain;
     lock->release = &fs_lock_release;
     lock->is_locked = &fs_lock_is_locked;
+    lock->rlock = Qnil;
     return lock;
 }
 

data/ext/isomorfeus_ferret_ext/frt_global.c

@@ -1,5 +1,6 @@
 #include "frt_global.h"
 #include "frt_hash.h"
+#include "frt_search.h"
 #include <stdarg.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -11,21 +12,33 @@
 #include <sys/types.h>
 #include <unistd.h>
 #include <signal.h>
+#include <ruby/encoding.h>
 
 const char *FRT_EMPTY_STRING = "";
 
-int frt_scmp(const void *p1, const void *p2)
-{
+rb_encoding *utf8_encoding;
+int utf8_mbmaxlen;
+OnigCodePoint cp_apostrophe;
+OnigCodePoint cp_dot;
+OnigCodePoint cp_comma;
+OnigCodePoint cp_backslash;
+OnigCodePoint cp_slash;
+OnigCodePoint cp_underscore;
+OnigCodePoint cp_dash;
+OnigCodePoint cp_hyphen;
+OnigCodePoint cp_at;
+OnigCodePoint cp_ampersand;
+OnigCodePoint cp_colon;
+
+int frt_scmp(const void *p1, const void *p2) {
     return strcmp(*(char **) p1, *(char **) p2);
 }
 
-void frt_strsort(char **str_array, int size)
-{
+void frt_strsort(char **str_array, int size) {
     qsort(str_array, size, sizeof(char *), &frt_scmp);
 }
 
-int frt_icmp(const void *p1, const void *p2)
-{
+int frt_icmp(const void *p1, const void *p2) {
     int i1 = *(int *) p1;
     int i2 = *(int *) p2;
 
@@ -38,42 +51,36 @@ int frt_icmp(const void *p1, const void *p2)
     return 0;
 }
 
-int frt_icmp_risky(const void *p1, const void *p2)
-{
+int frt_icmp_risky(const void *p1, const void *p2) {
   return (*(int *)p1) - *((int *)p2);
 }
 
-unsigned int *frt_imalloc(unsigned int value)
-{
+unsigned int *frt_imalloc(unsigned int value) {
   unsigned int *p = FRT_ALLOC(unsigned int);
   *p = value;
   return p;
 }
 
-unsigned long *frt_lmalloc(unsigned long value)
-{
+unsigned long *frt_lmalloc(unsigned long value) {
   unsigned long *p = FRT_ALLOC(unsigned long);
   *p = value;
   return p;
 }
 
-frt_u32 *frt_u32malloc(frt_u32 value)
-{
+frt_u32 *frt_u32malloc(frt_u32 value) {
   frt_u32 *p = FRT_ALLOC(frt_u32);
   *p = value;
   return p;
 }
 
-frt_u64 *frt_u64malloc(frt_u64 value)
-{
+frt_u64 *frt_u64malloc(frt_u64 value) {
   frt_u64 *p = FRT_ALLOC(frt_u64);
   *p = value;
   return p;
 }
 
 /* concatenate two strings freeing the second */
-char *frt_estrcat(char *str1, char *str2)
-{
+char *frt_estrcat(char *str1, char *str2) {
     size_t len1 = strlen(str1);
     size_t len2 = strlen(str2);
     FRT_REALLOC_N(str1, char, len1 + len2 + 3);     /* leave room for <CR> */
@@ -83,8 +90,7 @@ char *frt_estrcat(char *str1, char *str2)
 }
 
 /* epstrdup: duplicate a string with a format, report if error */
-char *frt_epstrdup(const char *fmt, int len, ...)
-{
+char *frt_epstrdup(const char *fmt, int len, ...) {
     char *string;
     va_list args;
     len += (int) strlen(fmt);
@@ -98,8 +104,7 @@ char *frt_epstrdup(const char *fmt, int len, ...)
 }
 
 /* frt_estrdup: duplicate a string, report if error */
-char *frt_estrdup(const char *s)
-{
+char *frt_estrdup(const char *s) {
     char *t = FRT_ALLOC_N(char, strlen(s) + 1);
     strcpy(t, s);
     return t;
@@ -108,15 +113,13 @@ char *frt_estrdup(const char *s)
 /* Pretty print a float to the buffer. The buffer should have at least 32
  * bytes available.
  */
-char *frt_dbl_to_s(char *buf, double num)
-{
+char *frt_dbl_to_s(char *buf, double num) {
     char *p, *e;
 
 #ifdef FRT_IS_C99
     if (isinf(num)) {
         return frt_estrdup(num < 0 ? "-Infinity" : "Infinity");
-    }
-    else if (isnan(num)) {
+    } else if (isnan(num)) {
         return frt_estrdup("NaN");
     }
 #endif
@@ -154,8 +157,7 @@ char *frt_strapp(char *dst, const char *src) {
 }
 
 /* strfmt: like sprintf except that it allocates memory for the string */
-char *frt_vstrfmt(const char *fmt, va_list args)
-{
+char *frt_vstrfmt(const char *fmt, va_list args) {
     char *string;
     char *p = (char *) fmt, *q;
     int len = (int) strlen(fmt) + 1;
@@ -217,8 +219,7 @@ char *frt_vstrfmt(const char *fmt, va_list args)
     return string;
 }
 
-char *frt_strfmt(const char *fmt, ...)
-{
+char *frt_strfmt(const char *fmt, ...) {
     va_list args;
     char *str;
     va_start(args, fmt);
@@ -227,8 +228,7 @@ char *frt_strfmt(const char *fmt, ...)
     return str;
 }
 
-void frt_dummy_free(void *p)
-{
+void frt_dummy_free(void *p) {
     (void)p; /* suppress unused argument warning */
 }
 
@@ -236,8 +236,7 @@ void frt_dummy_free(void *p)
 #define CMD_BUF_SIZE (128 + FILENAME_MAX)
 /* need to declare this as it is masked by default in linux */
 
-static char *build_shell_command()
-{
+static char *build_shell_command(void) {
     int   pid = getpid();
     char *buf = FRT_ALLOC_N(char, CMD_BUF_SIZE);
     char *command =
@@ -252,8 +251,7 @@ static char *build_shell_command()
 /**
  * Call out to gdb to get our stacktrace.
  */
-char *frt_get_stacktrace()
-{
+char *frt_get_stacktrace(void) {
 #ifdef HAVE_GDB
     FILE *stream;
     char *gdb_filename = NULL, *buf = NULL, *stack = NULL;
@@ -288,20 +286,18 @@ char *frt_get_stacktrace()
 #endif
 }
 
-void frt_print_stacktrace()
-{
+void frt_print_stacktrace(void) {
     char *stack = frt_get_stacktrace();
 
     if (stack) {
         fprintf(EXCEPTION_STREAM, "Stack trace:\n%s", stack);
         free(stack);
     } else {
-        fprintf(EXCEPTION_STREAM, "Stack trace not avaialble\n");
+        fprintf(EXCEPTION_STREAM, "Stack trace not available\n");
     }
 }
 
-typedef struct FreeMe
-{
+typedef struct FreeMe {
     void *p;
     frt_free_ft free_func;
 } FreeMe;
@@ -310,8 +306,7 @@ static FreeMe *free_mes = NULL;
 static int free_mes_size = 0;
 static int free_mes_capa = 0;
 
-void frt_register_for_cleanup(void *p, frt_free_ft free_func)
-{
+void frt_register_for_cleanup(void *p, frt_free_ft free_func) {
     FreeMe *free_me;
     if (free_mes_capa == 0) {
         free_mes_capa = 16;
@@ -330,18 +325,15 @@ void frt_register_for_cleanup(void *p, frt_free_ft free_func)
 static char name[MAX_PROG_NAME]; /* program name for error msgs */
 
 /* frt_setprogname: set stored name of program */
-void frt_setprogname(const char *str)
-{
+void frt_setprogname(const char *str) {
     strncpy(name, str, sizeof(name) - 1);
 }
 
-const char *frt_progname()
-{
+const char *frt_progname(void) {
     return name;
 }
 
-static const char *signal_to_string(int signum)
-{
+static const char *signal_to_string(int signum) {
     switch (signum)
     {
         case SIGILL:  return "SIGILL";
@@ -356,8 +348,7 @@ static const char *signal_to_string(int signum)
     return "Unknown Signal";
 }
 
-static void sighandler_crash(int signum)
-{
+static void sighandler_crash(int signum) {
     frt_print_stacktrace();
     FRT_XEXIT("Signal", "Exiting on signal %s (%d)", signal_to_string(signum), signum);
 }
@@ -366,8 +357,7 @@ static void sighandler_crash(int signum)
     signal(sig, handler);                  \
 } while(0)
 
-void frt_init(int argc, const char *const argv[])
-{
+void frt_init(int argc, const char *const argv[]) {
     if (argc > 0) {
         frt_setprogname(argv[0]);
     }
@@ -381,6 +371,63 @@ void frt_init(int argc, const char *const argv[])
     SETSIG_IF_UNSET(SIGSEGV, sighandler_crash);
 
     atexit(&frt_hash_finalize);
+
+    utf8_encoding = rb_enc_find("UTF-8");
+    utf8_mbmaxlen = rb_enc_mbmaxlen(utf8_encoding);
+    char *p = "'";
+    cp_apostrophe = rb_enc_mbc_to_codepoint(p, p + 1, utf8_encoding);
+    p = ".";
+    cp_dot = rb_enc_mbc_to_codepoint(p, p + 1, utf8_encoding);
+    p = ",";
+    cp_comma = rb_enc_mbc_to_codepoint(p, p + 1, utf8_encoding);
+    p = "\\";
+    cp_backslash = rb_enc_mbc_to_codepoint(p, p + 1, utf8_encoding);
+    p = "/";
+    cp_slash = rb_enc_mbc_to_codepoint(p, p + 1, utf8_encoding);
+    p = "_";
+    cp_underscore = rb_enc_mbc_to_codepoint(p, p + 1, utf8_encoding);
+    p = "-";
+    cp_dash = rb_enc_mbc_to_codepoint(p, p + 1, utf8_encoding);
+    p = "\u2010";
+    cp_hyphen = rb_enc_mbc_to_codepoint(p, p + 1, utf8_encoding);
+    p = "@";
+    cp_at = rb_enc_mbc_to_codepoint(p, p + 1, utf8_encoding);
+    p = "&";
+    cp_ampersand = rb_enc_mbc_to_codepoint(p, p + 1, utf8_encoding);
+    p = ":";
+    cp_colon = rb_enc_mbc_to_codepoint(p, p + 1, utf8_encoding);
+
+    FRT_SORT_FIELD_SCORE = frt_sort_field_alloc();
+    FRT_SORT_FIELD_SCORE->field_index_class = NULL;               /* field_index_class */
+    FRT_SORT_FIELD_SCORE->field = (ID)NULL;                       /* field */
+    FRT_SORT_FIELD_SCORE->type = FRT_SORT_TYPE_SCORE;             /* type */
+    FRT_SORT_FIELD_SCORE->reverse = false;                        /* reverse */
+    FRT_SORT_FIELD_SCORE->compare = frt_sort_field_score_compare; /* compare */
+    FRT_SORT_FIELD_SCORE->get_val = frt_sort_field_score_get_val; /* get_val */
+
+    FRT_SORT_FIELD_SCORE_REV = frt_sort_field_alloc();
+    FRT_SORT_FIELD_SCORE_REV->field_index_class = NULL;               /* field_index_class */
+    FRT_SORT_FIELD_SCORE_REV->field = (ID)NULL;                       /* field */
+    FRT_SORT_FIELD_SCORE_REV->type = FRT_SORT_TYPE_SCORE;             /* type */
+    FRT_SORT_FIELD_SCORE_REV->reverse = true;                         /* reverse */
+    FRT_SORT_FIELD_SCORE_REV->compare = frt_sort_field_score_compare; /* compare */
+    FRT_SORT_FIELD_SCORE_REV->get_val = frt_sort_field_score_get_val; /* get_val */
+
+    FRT_SORT_FIELD_DOC = frt_sort_field_alloc();
+    FRT_SORT_FIELD_DOC->field_index_class = NULL;             /* field_index_class */
+    FRT_SORT_FIELD_DOC->field = (ID)NULL;                     /* field */
+    FRT_SORT_FIELD_DOC->type = FRT_SORT_TYPE_DOC;             /* type */
+    FRT_SORT_FIELD_DOC->reverse = false;                      /* reverse */
+    FRT_SORT_FIELD_DOC->compare = frt_sort_field_doc_compare; /* compare */
+    FRT_SORT_FIELD_DOC->get_val = frt_sort_field_doc_get_val; /* get_val */
+
+    FRT_SORT_FIELD_DOC_REV = frt_sort_field_alloc();
+    FRT_SORT_FIELD_DOC_REV->field_index_class = NULL;             /* field_index_class */
+    FRT_SORT_FIELD_DOC_REV->field = (ID)NULL;                     /* field */
+    FRT_SORT_FIELD_DOC_REV->type = FRT_SORT_TYPE_DOC;             /* type */
+    FRT_SORT_FIELD_DOC_REV->reverse = true;                       /* reverse */
+    FRT_SORT_FIELD_DOC_REV->compare = frt_sort_field_doc_compare; /* compare */
+    FRT_SORT_FIELD_DOC_REV->get_val = frt_sort_field_doc_get_val; /* get_val */
 }
 
 /**
@@ -392,8 +439,7 @@ void frt_init(int argc, const char *const argv[])
 static bool p_switch = false;
 static bool p_switch_tmp = false;
 
-void p(const char *format, ...)
-{
+void p(const char *format, ...) {
     va_list args;
 
     if (!p_switch) return;
@@ -403,25 +449,21 @@ void p(const char *format, ...)
     va_end(args);
 }
 
-void p_on()
-{
+void p_on(void) {
     fprintf(stderr, "> > > > > STARTING PRINT\n");
     p_switch = true;
 }
 
-void p_off()
-{
+void p_off(void) {
     fprintf(stderr, "< < < < < STOPPING PRINT\n");
     p_switch = false;
 }
 
-void frt_p_pause()
-{
+void frt_p_pause(void) {
     p_switch_tmp = p_switch;
     p_switch = false;
 }
 
-void frt_p_resume()
-{
+void frt_p_resume(void) {
     p_switch = p_switch_tmp;
 }

data/ext/isomorfeus_ferret_ext/frt_global.h

@@ -11,12 +11,17 @@
 #include <string.h>
 #include <ruby.h>
 
-typedef ID FrtSymbol;
-
 #define FRT_MAX_WORD_SIZE 255
 #define FRT_MAX_FILE_PATH 1024
 #define FRT_BUFFER_SIZE 1024
 
+typedef enum {
+    FRT_COMPRESSION_NONE = 0,
+    FRT_COMPRESSION_BROTLI = 1,
+    FRT_COMPRESSION_BZ2 = 2,
+    FRT_COMPRESSION_LZ4 = 3
+} FrtCompressionType;
+
 #define FRT_DBL2S "%#.7g"
 
 #if __GNUC__ >= 3
@@ -37,7 +42,6 @@ typedef void (*frt_free_ft)(void *key);
 
 #define FRT_NELEMS(array) ((int)(sizeof(array)/sizeof(array[0])))
 
-
 #define FRT_ZEROSET(ptr, type) memset(ptr, 0, sizeof(type))
 #define FRT_ZEROSET_N(ptr, type, n) memset(ptr, 0, sizeof(type)*(n))
 

data/ext/isomorfeus_ferret_ext/frt_hash.c

@@ -510,8 +510,7 @@ void frt_h_str_print_keys(FrtHash *self, FILE *out)
     free(keys);
 }
 
-void frt_hash_finalize()
-{
+void frt_hash_finalize(void) {
     while (num_free_hts > 0) {
         free(free_hts[--num_free_hts]);
     }