isomorfeus-ferret 0.17.2 → 0.17.3

data/ext/isomorfeus_ferret_ext/frt_similarity.c

@@ -20,20 +20,17 @@ FrtTerm *frt_term_new(ID field, const char *text) {
     return t;
 }
 
-void frt_term_destroy(FrtTerm *self)
-{
+void frt_term_destroy(FrtTerm *self) {
     free(self->text);
     free(self);
 }
 
-int frt_term_eq(const void *t1, const void *t2)
-{
+int frt_term_eq(const void *t1, const void *t2) {
     return ((strcmp(((FrtTerm *)t1)->text, ((FrtTerm *)t2)->text) == 0) &&
         (((FrtTerm *)t1)->field == ((FrtTerm *)t2)->field));
 }
 
-unsigned long long frt_term_hash(const void *t)
-{
+unsigned long long frt_term_hash(const void *t) {
     return frt_str_hash(((FrtTerm *)t)->text) * frt_str_hash(rb_id2name(((FrtTerm *)t)->field));
 }
 
@@ -49,20 +46,17 @@ static float simdef_length_norm(FrtSimilarity *s, ID field, int num_terms) {
     return (float)(1.0 / sqrt(num_terms));
 }
 
-static float simdef_query_norm(struct FrtSimilarity *s, float sum_of_squared_weights)
-{
+static float simdef_query_norm(struct FrtSimilarity *s, float sum_of_squared_weights) {
     (void)s;
     return (float)(1.0 / sqrt(sum_of_squared_weights));
 }
 
-static float simdef_tf(struct FrtSimilarity *s, float freq)
-{
+static float simdef_tf(struct FrtSimilarity *s, float freq) {
     (void)s;
     return (float)sqrt(freq);
 }
 
-static float simdef_sloppy_freq(struct FrtSimilarity *s, int distance)
-{
+static float simdef_sloppy_freq(struct FrtSimilarity *s, int distance) {
     (void)s;
     return (float)(1.0 / (double)(distance + 1));
 }
@@ -84,31 +78,26 @@ static float simdef_idf_phrase(struct FrtSimilarity *s, ID field, FrtPhrasePosit
     return idf;
 }
 
-static float simdef_idf(struct FrtSimilarity *s, int doc_freq, int num_docs)
-{
+static float simdef_idf(struct FrtSimilarity *s, int doc_freq, int num_docs) {
     (void)s;
     return (float)(log((float)num_docs/(float)(doc_freq+1)) + 1.0);
 }
 
-static float simdef_coord(struct FrtSimilarity *s, int overlap, int max_overlap)
-{
+static float simdef_coord(struct FrtSimilarity *s, int overlap, int max_overlap) {
     (void)s;
     return (float)((double)overlap / (double)max_overlap);
 }
 
-static float simdef_decode_norm(struct FrtSimilarity *s, frt_uchar b)
-{
+static float simdef_decode_norm(struct FrtSimilarity *s, frt_uchar b) {
     return s->norm_table[b];
 }
 
-static frt_uchar simdef_encode_norm(struct FrtSimilarity *s, float f)
-{
+static frt_uchar simdef_encode_norm(struct FrtSimilarity *s, float f) {
     (void)s;
     return frt_float2byte(f);
 }
 
-static void simdef_destroy(FrtSimilarity *s)
-{
+static void simdef_destroy(FrtSimilarity *s) {
     (void)s;
     /* nothing to do here */
 }

data/ext/isomorfeus_ferret_ext/frt_similarity.h

@@ -25,8 +25,7 @@ extern unsigned long long frt_term_hash(const void *t);
  *
  ***************************************************************************/
 
-typedef struct FrtPhrasePosition
-{
+typedef struct FrtPhrasePosition {
     int pos;
     char **terms;
 } FrtPhrasePosition;

data/ext/isomorfeus_ferret_ext/frt_store.c

@@ -1,9 +1,6 @@
 #include "frt_store.h"
 #include <string.h>
 
-
-
-
 /*
  * TODO: add try finally
  */
@@ -18,8 +15,8 @@ void frt_with_lock(FrtLock *lock, void (*func)(void *arg), void *arg) {
 /*
  * TODO: add try finally
  */
-void frt_with_lock_name(FrtStore *store, const char *lock_name, void (*func)(void *arg), void *arg) {
-    FrtLock *lock = store->open_lock_i(store, lock_name);
+void frt_with_lock_name(FrtStore *store, const char *folder_name, const char *lock_name, void (*func)(void *arg), void *arg) {
+    FrtLock *lock = store->open_lock_i(store, folder_name, lock_name);
     if (!lock->obtain(lock)) {
         FRT_RAISE(FRT_LOCK_ERROR, "couldn't obtain lock \"%s\"", lock->name);
     }
@@ -28,20 +25,17 @@ void frt_with_lock_name(FrtStore *store, const char *lock_name, void (*func)(voi
     store->close_lock_i(lock);
 }
 
-FrtLock *frt_open_lock(FrtStore *store, const char *lockname)
-{
-    FrtLock *lock = store->open_lock_i(store, lockname);
+FrtLock *frt_open_lock(FrtStore *store, const char *folder_name, const char *lockname) {
+    FrtLock *lock = store->open_lock_i(store, folder_name, lockname);
     frt_hs_add(store->locks, lock);
     return lock;
 }
 
-void frt_close_lock(FrtLock *lock)
-{
+void frt_close_lock(FrtLock *lock) {
     frt_hs_del(lock->store->locks, lock);
 }
 
-static void frt_close_lock_i(FrtLock *lock)
-{
+static void frt_close_lock_i(FrtLock *lock) {
     lock->store->close_lock_i(lock);
 }
 
@@ -94,14 +88,12 @@ void frt_store_close(FrtStore *store) {
  * @param filename the name of the file to check
  * @return 1 (true) if the file is a lock file, 0 (false) otherwise
  */
-int frt_file_is_lock(const char *filename)
-{
+int frt_file_is_lock(const char *filename) {
     int start = (int) strlen(filename) - 4;
     return ((start > 0) && (strcmp(FRT_LOCK_EXT, &filename[start]) == 0));
 }
 
-void frt_is2os_copy_bytes(FrtInStream *is, FrtOutStream *os, int cnt)
-{
+void frt_is2os_copy_bytes(FrtInStream *is, FrtOutStream *os, int cnt) {
     int len;
     frt_uchar buf[FRT_BUFFER_SIZE];
 
@@ -112,8 +104,7 @@ void frt_is2os_copy_bytes(FrtInStream *is, FrtOutStream *os, int cnt)
     }
 }
 
-void frt_is2os_copy_vints(FrtInStream *is, FrtOutStream *os, int cnt)
-{
+void frt_is2os_copy_vints(FrtInStream *is, FrtOutStream *os, int cnt) {
     frt_uchar b;
     for (; cnt > 0; cnt--) {
         while (((b = frt_is_read_byte(is)) & 0x80) != 0) {
@@ -126,8 +117,7 @@ void frt_is2os_copy_vints(FrtInStream *is, FrtOutStream *os, int cnt)
 /**
  * Test argument used to test the store->each function
  */
-struct FileNameListArg
-{
+struct FileNameListArg {
     int count;
     int size;
     int total_len;
@@ -137,8 +127,7 @@ struct FileNameListArg
 /**
  * Test function used to test store->each function
  */
-static void add_file_name(const char *fname, void *arg)
-{
+static void add_file_name(const char *fname, void *arg) {
     struct FileNameListArg *fnl = (struct FileNameListArg *)arg;
     if (fnl->count >= fnl->size) {
         fnl->size *= 2;
@@ -148,8 +137,7 @@ static void add_file_name(const char *fname, void *arg)
     fnl->total_len += strlen(fname) + 2;
 }
 
-char *frt_store_to_s(FrtStore *store)
-{
+char *frt_store_folder_to_s(FrtStore *store, const char *folder_name) {
     struct FileNameListArg fnl;
     char *buf, *b;
     int i;
@@ -158,7 +146,7 @@ char *frt_store_to_s(FrtStore *store)
     fnl.total_len = 10;
     fnl.files = FRT_ALLOC_N(char *, 16);
 
-    store->each(store, &add_file_name, &fnl);
+    store->each(store, folder_name, &add_file_name, &fnl);
     qsort(fnl.files, fnl.count, sizeof(char *), &frt_scmp);
     b = buf = FRT_ALLOC_N(char, fnl.total_len);
 

data/ext/isomorfeus_ferret_ext/frt_store.h

@@ -21,7 +21,8 @@ typedef struct FrtStore FrtStore;
 typedef struct FrtLock FrtLock;
 
 struct FrtLock {
-    char *name;
+    const char *folder_name;
+    const char *name;
     FrtStore *store;
     int (*obtain)(FrtLock *lock);
     int (*is_locked)(FrtLock *lock);
@@ -30,8 +31,9 @@ struct FrtLock {
 };
 
 typedef struct MDBXInfo {
-    MDBX_env *env;
-    char     *path;
+    MDBX_env   *env;
+    const char *path;
+    FrtStore   *temp_store;
 } MDBXInfo;
 
 struct FrtStore {
@@ -39,7 +41,6 @@ struct FrtStore {
     pthread_mutex_t      mutex_i;        /* for internal use only */
     pthread_mutex_t      mutex;          /* external mutex for use outside */
     union {
-        char             *path; /* for fs_store only */
         FrtHash          *ht;   /* for ram_store only */
         MDBXInfo         *mdbx;  /* for mdbx store only */
     } dir;
@@ -51,52 +52,76 @@ struct FrtStore {
     FrtHashSet  *locks;
 
     /**
-     * Create the file +filename+ in the +store+.
+     * Create the folder +folder_name+ in the store if it
+     * doesn't exist.
      *
-     * @param store self
+     * @param self store
+     * @param folder_name the name of the folder to create
+     * @raise FRT_IO_ERROR if the folder cannot be created
+     */
+    void (*create_folder)(FrtStore *self, const char *folder_name);
+
+    /**
+     * Remove the folder +folder_name+ from the store.
+     *
+     * @param self store
+     * @param folder_name the name of the folder to remove
+     * @raise FRT_IO_ERROR if the folder cannot be created
+     */
+    void (*remove_folder)(FrtStore *self, const char *folder_name);
+
+    /**
+     * Create the file +filename+ in the store.
+     *
+     * @param self store
+     * @param folder_name the name of the folder
      * @param filename the name of the file to create
      * @raise FRT_IO_ERROR if the file cannot be created
      */
-    void (*touch)(FrtStore *store, const char *filename);
+    void (*touch)(FrtStore *self, const char *folder_name, const char *filename);
 
     /**
-     * Return true if a file of name +filename+ exists in +store+.
+     * Return true if a file of name +filename+ exists in store.
      *
-     * @param store self
+     * @param self store
+     * @param folder_name the name of the folder
      * @param filename the name of the file to check for
      * @returns true if the file exists
-     * @raise FRT_IO_ERROR if there is an error checking for the files existance
+     * @raise FRT_IO_ERROR if there is an error checking for the files existence
      */
-    int (*exists)(FrtStore *store, const char *filename);
+    int (*exists)(FrtStore *self, const char *folder_name, const char *filename);
 
     /**
-     * Remove the file +filename+ from the +store+
+     * Remove the file +filename+ from the store.
      *
-     * @param store self
+     * @param self store
+     * @param folder_name the name of the folder
      * @param filename the name of the file to remove
      * @returns On success, zero is returned.  On error, -1 is returned, and errno
      *          is set appropriately.
      */
-    int (*remove)(FrtStore *store, const char *filename);
+    int (*remove)(FrtStore *self, const char *folder_name, const char *filename);
 
     /**
-     * Rename the file in the +store+ from the name +from+ to the name +to+.
+     * Rename the file in the store from the name +from+ to the name +to+.
      *
-     * @param store self
+     * @param self store
+     * @param folder_name the name of the folder
      * @param from the name of the file to rename
      * @param to the new name of the file
      * @raise FRT_IO_ERROR if there is an error renaming the file
      */
-    void (*rename)(FrtStore *store, const char *from, const char *to);
+    void (*rename)(FrtStore *self, const char *folder_name, const char *from, const char *to);
 
     /**
      * Returns the number of files in the store.
      *
-     * @param store self
+     * @param self store
+     * @param folder_name the name of the folder
      * @return the number of files in the store
      * @raise FRT_IO_ERROR if there is an error opening the directory
      */
-    int (*count)(FrtStore *store);
+    int (*count)(FrtStore *self, const char *folder_name);
 
     /**
      * Call the function +func+ with each filename in the store and the arg
@@ -104,80 +129,87 @@ struct FrtStore {
      * as the argument. If you need to pass more than one argument, you should
      * pass a struct.
      *
-     * @param store self
+     * @param self store
+     * @param folder_name the name of the folder
      * @param func the function to call with each files name and the +arg+
      *   passed
      * @param arg the argument to pass to the function
      * @raise FRT_IO_ERROR if there is an error opening the directory
      */
-    void (*each)(FrtStore *store, void (*func)(const char *fname, void *arg),
-                  void *arg);
+    void (*each)(FrtStore *self, const char *folder_name, void (*func)(const char *fname, void *arg), void *arg);
 
     /**
      * Clear all the locks in the store.
      *
-     * @param store self
+     * @param self store
+     * @param folder_name the name of the folder
      * @raise FRT_IO_ERROR if there is an error opening the directory
      */
-    void (*clear_locks)(FrtStore *store);
+    void (*clear_locks)(FrtStore *self, const char *folder_name);
 
     /**
      * Clear all files from the store except the lock files.
      *
-     * @param store self
+     * @param self store
+     * @param folder_name the name of the folder
      * @raise FRT_IO_ERROR if there is an error deleting the files
      */
-    void (*clear)(FrtStore *store);
+    void (*clear)(FrtStore *self, const char *folder_name);
 
     /**
      * Clear all files from the store including the lock files.
      *
-     * @param store self
+     * @param self store
+     * @param folder_name the name of the folder
      * @raise FRT_IO_ERROR if there is an error deleting the files
      */
-    void (*clear_all)(FrtStore *store);
+    void (*clear_all)(FrtStore *self, const char *folder_name);
 
     /**
-     * Return the length of the file +filename+ in +store+
+     * Return the length of the file +filename+ in +store+.
      *
-     * @param store self
+     * @param self store
+     * @param folder_name the name of the folder
      * @param the name of the file to check the length of
      * @return the length of the file in bytes
      * @raise FRT_IO_ERROR if there is an error checking the file length
      */
-    frt_off_t (*length)(FrtStore *store, const char *filename);
+    frt_off_t (*length)(FrtStore *self, const char *folder_name, const char *filename);
 
     /**
      * Allocate the resources needed for the output stream in the +store+ with
-     * the name +filename+
+     * the name +filename+.
      *
-     * @param store self
+     * @param self store
+     * @param folder_name the name of the folder
      * @param filename the name of the output stream
-     * @return a newly allocated filestream
+     * @return a newly allocated file stream
      * @raise FRT_IO_ERROR if there is an error opening the output stream
      *   resources
      */
-    FrtOutStream *(*new_output)(FrtStore *store, const char *filename);
+    FrtOutStream *(*new_output)(FrtStore *self, const char *folder_name, const char *filename);
 
     /**
-     * Open an input stream in the +store+ with the name +filename+
+     * Open an input stream in the +store+ with the name +filename+.
      *
-     * @param store self
+     * @param self store
+     * @param folder_name the name of the folder
      * @param filename the name of the input stream
      * @raise FRT_FILE_NOT_FOUND_ERROR if the input stream cannot be opened
      */
-    FrtInStream *(*open_input)(FrtStore *store, const char *filename);
+    FrtInStream *(*open_input)(FrtStore *self, const char *folder_name, const char *filename);
 
     /**
-     * Obtain a lock on the lock +lock+
+     * Obtain a lock on the lock +lock+.
      *
-     * @param store self
+     * @param self store
+     * @param folder_name the name of the folder
      * @param lock the lock to obtain
      */
-    FrtLock *(*open_lock_i)(FrtStore *store, const char *lockname);
+    FrtLock *(*open_lock_i)(FrtStore *self, const char *folder_name, const char *lockname);
 
     /**
-     * Returns true if +lock+ is locked. To test if the file is locked:wq
+     * Returns true if +lock+ is locked. To test if the file is locked:wq.
      *
      * @param lock the lock to test
      * @raise FRT_IO_ERROR if there is an error detecting the lock status
@@ -188,9 +220,9 @@ struct FrtStore {
      * Internal function to close the store freeing implementation specific
      * resources.
      *
-     * @param store self
+     * @param self store
      */
-    void (*close_i)(FrtStore *store);
+    void (*close_i)(FrtStore *self);
 
     /**
      * Reference to be used within ruby realm.
@@ -220,12 +252,13 @@ extern FrtStore *frt_open_ram_store(FrtStore *new_store);
  * method would be used for example to read an index into memory for faster
  * searching.
  *
- * @param store the whose contents will be copied into the newly allocated RAM
- *   store
- * @param close_store close the store whose contents where copied
+ * @param to_store the target store
+ * @param from_store the store, whose contents will be copied into the newly allocated RAM store
+ * @param folder_name folder to be copied, will be create in the target store, if it doesn't exist
+ * @param close_store close the from_store whose contents where copied
  * @return a newly allocated RAM FrtStore.
  */
-extern FrtStore *frt_open_ram_store_and_copy(FrtStore *store, FrtStore *from_store, bool close_store);
+extern FrtStore *frt_open_ram_store_and_copy(FrtStore *to_store, FrtStore *from_store, const char *folder_name, bool close_store);
 
 /*
  * == RamStore functions ==
@@ -294,7 +327,7 @@ extern void frt_with_lock(FrtLock *lock, void (*func)(void *arg), void *arg);
  * @raise FRT_IO_ERROR  if the lock is already locked
  * @see frt_with_lock
  */
-extern void frt_with_lock_name(FrtStore *store, const char *lock_name, void (*func)(void *arg), void *arg);
+extern void frt_with_lock_name(FrtStore *store, const char *folder_name, const char *lock_name, void (*func)(void *arg), void *arg);
 
 /**
  * Remove a reference to the store. If the reference count gets to zero free
@@ -325,13 +358,14 @@ extern void frt_is2os_copy_bytes(FrtInStream *is, FrtOutStream *os, int cnt);
 extern void frt_is2os_copy_vints(FrtInStream *is, FrtOutStream *os, int cnt);
 
 /**
- * Print the filenames in a store to a buffer.
+ * Print the filenames in a store folder to a buffer.
  *
  * @param store the store to get the filenames from
+ * @param folder_name the folder to get the filenames from
  */
-extern char *frt_store_to_s(FrtStore *store);
+extern char *frt_store_folder_to_s(FrtStore *store, const char *folder_name);
 
-extern FrtLock *frt_open_lock(FrtStore *store, const char *lockname);
+extern FrtLock *frt_open_lock(FrtStore *store, const char *folder_name, const char *lockname);
 extern void frt_close_lock(FrtLock *lock);
 
 /* required by submodules

data/ext/isomorfeus_ferret_ext/frt_term_vectors.c

@@ -10,8 +10,7 @@
  *
  ****************************************************************************/
 
-void frt_tv_destroy(FrtTermVector *tv)
-{
+void frt_tv_destroy(FrtTermVector *tv) {
     int i = tv->term_cnt;
     while (i > 0) {
         i--;
@@ -23,8 +22,7 @@ void frt_tv_destroy(FrtTermVector *tv)
     free(tv);
 }
 
-int frt_tv_scan_to_term_index(FrtTermVector *tv, const char *term)
-{
+int frt_tv_scan_to_term_index(FrtTermVector *tv, const char *term) {
     int lo = 0;                 /* search starts array */
     int hi = tv->term_cnt - 1;  /* for 1st element < n, return its index */
     int mid;
@@ -37,36 +35,30 @@ int frt_tv_scan_to_term_index(FrtTermVector *tv, const char *term)
         cmp = strcmp(term, mid_term);
         if (cmp < 0) {
             hi = mid - 1;
-        }
-        else if (cmp > 0) {
+        } else if (cmp > 0) {
             lo = mid + 1;
-        }
-        else {                  /* found a match */
+        } else {                  /* found a match */
             return mid;
         }
     }
     return lo;
 }
 
-int frt_tv_get_term_index(FrtTermVector *tv, const char *term)
-{
+int frt_tv_get_term_index(FrtTermVector *tv, const char *term) {
     int index = frt_tv_scan_to_term_index(tv, term);
     if (index < tv->term_cnt && (0 == strcmp(term, tv->terms[index].text))) {
         /* found term */
         return index;
-    }
-    else {
+    } else {
         return -1;
     }
 }
 
-FrtTVTerm *frt_tv_get_tv_term(FrtTermVector *tv, const char *term)
-{
+FrtTVTerm *frt_tv_get_tv_term(FrtTermVector *tv, const char *term) {
     int index = frt_tv_get_term_index(tv, term);
     if (index >= 0) {
         return &(tv->terms[index]);
-    }
-    else {
+    } else {
         return NULL;
     }
 }

data/ext/isomorfeus_ferret_ext/frt_win32.h

@@ -4,20 +4,17 @@
 #include "frt_global.h"
 #include <io.h>
 
-struct dirent
-{
+struct dirent {
     char *d_name;
 };
 
-typedef struct DIR
-{
+typedef struct DIR {
     struct _finddata_t find_data;
     struct dirent de;
     long handle;
 } DIR;
 
-DIR *opendir(const char *dirname)
-{
+DIR *opendir(const char *dirname) {
     DIR *d = FRT_ALLOC_AND_ZERO(DIR);
     char dirname_buf[FRT_MAX_FILE_PATH];
     long ff_res;
@@ -33,8 +30,7 @@ DIR *opendir(const char *dirname)
     return d;
 }
 
-struct dirent *readdir(DIR *d)
-{
+struct dirent *readdir(DIR *d) {
     /* _findfirst already returned so do _findnext */
     if (d->de.d_name != NULL) {
         if (_findnext(d->handle, &d->find_data) < 0) {
@@ -45,8 +41,7 @@ struct dirent *readdir(DIR *d)
     return &d->de;
 }
 
-void closedir(DIR *d)
-{
+void closedir(DIR *d) {
     _findclose(d->handle);
     free(d);
 }

data/ext/isomorfeus_ferret_ext/isomorfeus_ferret.c

@@ -5,6 +5,8 @@
 #include "frt_hashset.h"
 #include "frt_threading.h"
 
+const char *segm_idx_name = "segmented_index";
+
 /* IDs */
 ID id_new;
 ID id_call;
@@ -32,21 +34,21 @@ VALUE sym_each;
 /* Modules */
 VALUE mIsomorfeus;
 VALUE mFerret;
-VALUE mStore;
-VALUE mStringHelper;
-VALUE mSpans;
 
 /* Classes */
-VALUE cTerm;
+static VALUE cTerm;
 VALUE cFileNotFoundError;
 VALUE cParseError;
 VALUE cStateError;
 
-void Init_Benchmark(void);
-void Init_Test(void);
+extern void Init_Benchmark(void);
+extern void Init_Test(void);
 
 unsigned long value_hash(const void *key) {
-    return (unsigned long)key;
+    /* double cast to prevent compiler warning:
+     * - first cast: pointer to number of same size
+     * - second cast: number to smaller number */
+    return (unsigned long)(uintptr_t)key;
 }
 
 int value_eq(const void *key1, const void *key2) {
@@ -119,15 +121,14 @@ ID frb_field(VALUE rfield) {
  * Jeremie 'ahFeel' BORDIER
  * ahFeel@rift.Fr
  */
-char *json_concat_string(char *s, char *field) {
+char *json_concat_string(char *s, const char *field) {
     *(s++) = '"';
 	while (*field) {
 		if (*field == '"') {
             *(s++) = '\'';
             *(s++) = *(field++);
             *(s++) = '\'';
-        }
-        else {
+        } else {
             *(s++) = *(field++);
         }
     }
@@ -308,6 +309,6 @@ void Init_isomorfeus_ferret_ext(void) {
     rb_define_const(mFerret, "FIX_INT_MAX", INT2FIX(INT_MAX >> 1));
 }
 
-extern void frb_raise(int excode, const char *msg) {
+void frb_raise(int excode, const char *msg) {
     rb_raise(frb_get_error(ERROR_TYPES[excode]), "%s", msg);
 }