isomorfeus-ferret 0.17.1 → 0.17.3

data/ext/isomorfeus_ferret_ext/frt_in_stream.h

@@ -0,0 +1,241 @@
+#ifndef FRT_IN_STREAM_H
+#define FRT_IN_STREAM_H
+
+#include "frt_config.h"
+#include "frt_global.h"
+#include "frt_stream.h"
+#include "frt_ram_store.h"
+
+typedef struct FrtInStream FrtInStream;
+
+struct FrtInStreamMethods {
+    /**
+     * Read +len+ characters from the input stream into the +offset+ position in
+     * +buf+, an array of unsigned characters.
+     *
+     * @param is self
+     * @param buf an array of characters which must be allocated with  at least
+     *          +offset+ + +len+ bytes
+     * @param len the number of bytes to read
+     * @raise FRT_IO_ERROR if there is an error reading from the input stream
+     */
+    void (*read_i)(struct FrtInStream *is, frt_uchar *buf, int len);
+
+    /**
+     * Seek position +pos+ in input stream +is+
+     *
+     * @param is self
+     * @param pos the position to seek
+     * @raise FRT_IO_ERROR if the seek fails
+     */
+    void (*seek_i)(struct FrtInStream *is, frt_off_t pos);
+
+    /**
+     * Returns the length of the input stream +is+
+     *
+     * @param is self
+     * @raise FRT_IO_ERROR if there is an error getting the file length
+     */
+    frt_off_t (*length_i)(struct FrtInStream *is);
+
+    /**
+     * Close the resources allocated to the inputstream +is+
+     *
+     * @param is self
+     * @raise FRT_IO_ERROR if the close fails
+     */
+    void (*close_i)(struct FrtInStream *is);
+};
+
+typedef struct FrtInStreamFile {
+    _Atomic unsigned int ref_cnt;
+    union {
+        int fd;
+        FrtRAMFile *rf;
+    } file;
+} FrtInStreamFile;
+
+struct FrtInStream {
+    const char *folder_name;
+    FrtBuffer buf;
+    struct FrtInStreamFile *f;
+    union {
+        frt_off_t pointer;      /* only used by RAMIn */
+        char *path;             /* only used by FSIn */
+    } d;
+    _Atomic unsigned int ref_cnt;
+    const struct FrtInStreamMethods *m;
+};
+
+/**
+ * Get the current position within an FrtInStream.
+ *
+ * @param is the FrtInStream to get the current position from
+ * @return the current position within the FrtInStream +is+
+ */
+extern frt_off_t frt_is_pos(FrtInStream *is);
+
+/**
+ * Set the current position in FrtInStream +is+ to +pos+.
+ *
+ * @param is the FrtInStream to set the current position in
+ * @param pos the position in FrtInStream to seek
+ * @raise FRT_IO_ERROR if there is a error seeking from the file-system
+ * @raise FRT_EOF_ERROR if there is an attempt to seek past the end of the file
+ */
+extern void frt_is_seek(FrtInStream *is, frt_off_t pos);
+
+/**
+ * Close the FrtInStream freeing all allocated resources.
+ *
+ * @param is the FrtInStream to close
+ * @raise FRT_IO_ERROR if there is an error closing the associated file
+ */
+extern void frt_is_close(FrtInStream *is);
+
+/**
+ * Clone the FrtInStream allocating a new FrtInStream structure
+ *
+ * @param is the FrtInStream to clone
+ * @return a newly allocated FrtInStream which is a clone of +is+
+ */
+extern FrtInStream *frt_is_clone(FrtInStream *is);
+
+/**
+ * Read a singly byte (unsigned char) from the FrtInStream +is+.
+ *
+ * @param is the Instream to read from
+ * @return a single unsigned char read from the FrtInStream +is+
+ * @raise FRT_IO_ERROR if there is a error reading from the file-system
+ * @raise FRT_EOF_ERROR if there is an attempt to read past the end of the file
+ */
+extern frt_uchar frt_is_read_byte(FrtInStream *is);
+
+/**
+ * Read +len+ bytes from FrtInStream +is+ and write them to buffer +buf+
+ *
+ * @param is     the FrtInStream to read from
+ * @param buf    the buffer to read into, that is copy the bytes read to
+ * @param len    the number of bytes to read
+ * @return       the resultant buffer +buf+
+ * @raise FRT_IO_ERROR if there is a error reading from the file-system
+ * @raise FRT_EOF_ERROR if there is an attempt to read past the end of the file
+ */
+extern frt_uchar *frt_is_read_bytes(FrtInStream *is, frt_uchar *buf, int len);
+
+/**
+ * Read a 32-bit unsigned integer from the FrtInStream.
+ *
+ * @param is the FrtInStream to read from
+ * @return a 32-bit unsigned integer
+ * @raise FRT_IO_ERROR if there is a error reading from the file-system
+ * @raise FRT_EOF_ERROR if there is an attempt to read past the end of the file
+ */
+extern frt_i32 frt_is_read_i32(FrtInStream *is);
+
+/**
+ * Read a 64-bit unsigned integer from the FrtInStream.
+ *
+ * @param is the FrtInStream to read from
+ * @return a 64-bit unsigned integer
+ * @raise FRT_IO_ERROR if there is a error reading from the file-system
+ * @raise FRT_EOF_ERROR if there is an attempt to read past the end of the file
+ */
+extern frt_i64 frt_is_read_i64(FrtInStream *is);
+
+/**
+ * Read a 32-bit signed integer from the FrtInStream.
+ *
+ * @param is the FrtInStream to read from
+ * @return a 32-bit signed integer
+ * @raise FRT_IO_ERROR if there is a error reading from the file-system
+ * @raise FRT_EOF_ERROR if there is an attempt to read past the end of the file
+ */
+extern frt_u32 frt_is_read_u32(FrtInStream *is);
+
+/**
+ * Read a 64-bit signed integer from the FrtInStream.
+ *
+ * @param is the FrtInStream to read from
+ * @return a 64-bit signed integer
+ * @raise FRT_IO_ERROR if there is a error reading from the file-system
+ * @raise FRT_EOF_ERROR if there is an attempt to read past the end of the file
+ */
+extern frt_u64 frt_is_read_u64(FrtInStream *is);
+
+/**
+ * Read a compressed (VINT) unsigned integer from the FrtInStream.
+ * TODO: describe VINT format
+ *
+ * @param is the FrtInStream to read from
+ * @return an int
+ * @raise FRT_IO_ERROR if there is a error reading from the file-system
+ * @raise FRT_EOF_ERROR if there is an attempt to read past the end of the file
+ */
+extern unsigned int frt_is_read_vint(FrtInStream *is);
+
+/**
+ * Skip _cnt_ vints. This is a convenience method used for performance reasons
+ * to skip large numbers of vints. It is mostly used by TermDocEnums. When
+ * skipping positions os the proximity index file.
+ *
+ * @param is the FrtInStream to read from
+ * @param cnt the number of vints to skip
+ * @raise FRT_IO_ERROR if there is a error reading from the file-system
+ * @raise FRT_EOF_ERROR if there is an attempt to read past the end of the file
+ */
+extern void frt_is_skip_vints(FrtInStream *is, register int cnt);
+
+/**
+ * Read a compressed (VINT) unsigned frt_off_t from the FrtInStream.
+ * TODO: describe VINT format
+ *
+ * @param is the FrtInStream to read from
+ * @return a frt_off_t
+ * @raise FRT_IO_ERROR if there is a error reading from the file-system
+ * @raise FRT_EOF_ERROR if there is an attempt to read past the end of the file
+ */
+extern frt_off_t frt_is_read_voff_t(FrtInStream *is);
+
+/**
+ * Read a compressed (VINT) unsigned 64bit int from the FrtInStream.
+ * TODO: describe VINT format
+ *
+ * @param is the FrtInStream to read from
+ * @return a 64bit int
+ * @raise FRT_IO_ERROR if there is a error reading from the file-system
+ * @raise FRT_EOF_ERROR if there is an attempt to read past the end of the file
+ */
+extern frt_u64 frt_is_read_vll(FrtInStream *is);
+
+/**
+ * Read a string from the FrtInStream. A string is an integer +length+ in vint
+ * format (see frt_is_read_vint) followed by +length+ bytes. This is the format
+ * used by frt_os_write_string.
+ *
+ * @param is the FrtInStream to read from
+ * @return a null byte delimited string
+ * @raise FRT_IO_ERROR if there is a error reading from the file-system
+ * @raise FRT_EOF_ERROR if there is an attempt to read past the end of the file
+ */
+extern char *frt_is_read_string(FrtInStream *is);
+
+/**
+ * Read a string from the FrtInStream. A string is an integer +length+ in vint
+ * format (see frt_is_read_vint) followed by +length+ bytes. This is the format
+ * used by frt_os_write_string. This method is similar to +frt_is_read_string+ except
+ * that it will safely free all memory if there is an error reading the
+ * string.
+ *
+ * @param is the FrtInStream to read from
+ * @return a null byte delimited string
+ * @raise FRT_IO_ERROR if there is a error reading from the file-system
+ * @raise FRT_EOF_ERROR if there is an attempt to read past the end of the file
+ */
+extern char *frt_is_read_string_safe(FrtInStream *is);
+
+extern char *frt_is_read_compressed_bytes(FrtInStream *is, int compressed_len, int *len, FrtCompressionType compression_type);
+
+#define frt_is_length(mis) mis->m->length_i(mis)
+
+#endif

data/ext/isomorfeus_ferret_ext/frt_ind.c

@@ -44,6 +44,7 @@ FrtIndex *frt_index_new(FrtStore *store, FrtAnalyzer *analyzer, FrtHashSet *def_
         FRT_REF(store);
     } else {
         self->store = frt_open_ram_store(NULL);
+        self->store->create_folder(self->store, segm_idx_name);
         create = true;
     }
     if (analyzer) {
@@ -79,8 +80,7 @@ FrtIndex *frt_index_new(FrtStore *store, FrtAnalyzer *analyzer, FrtHashSet *def_
     return self;
 }
 
-void frt_index_destroy(FrtIndex *self)
-{
+void frt_index_destroy(FrtIndex *self) {
     pthread_mutex_destroy(&self->mutex);
     INDEX_CLOSE_READER(self);
     if (self->iw) frt_iw_close(self->iw);
@@ -91,8 +91,7 @@ void frt_index_destroy(FrtIndex *self)
     free(self);
 }
 
-void frt_ensure_writer_open(FrtIndex *self)
-{
+void frt_ensure_writer_open(FrtIndex *self) {
     if (!self->iw) {
         INDEX_CLOSE_READER(self);
 
@@ -102,8 +101,7 @@ void frt_ensure_writer_open(FrtIndex *self)
     }
 }
 
-void frt_ensure_reader_open(FrtIndex *self)
-{
+void frt_ensure_reader_open(FrtIndex *self) {
     if (self->ir) {
         if (self->check_latest && !frt_ir_is_latest(self->ir)) {
             INDEX_CLOSE_READER(self);
@@ -118,16 +116,14 @@ void frt_ensure_reader_open(FrtIndex *self)
     self->ir = frt_ir_open(NULL, self->store);
 }
 
-void frt_ensure_searcher_open(FrtIndex *self)
-{
+void frt_ensure_searcher_open(FrtIndex *self) {
     frt_ensure_reader_open(self);
     if (!self->sea) {
         self->sea = frt_isea_new(self->ir);
     }
 }
 
-int frt_index_size(FrtIndex *self)
-{
+int frt_index_size(FrtIndex *self) {
     int size;
     pthread_mutex_lock(&self->mutex);
     {
@@ -138,19 +134,7 @@ int frt_index_size(FrtIndex *self)
     return size;
 }
 
-void frt_index_optimize(FrtIndex *self)
-{
-    pthread_mutex_lock(&self->mutex);
-    {
-        frt_ensure_writer_open(self);
-        frt_iw_optimize(self->iw);
-        AUTOFLUSH_IW(self);
-    }
-    pthread_mutex_unlock(&self->mutex);
-}
-
-bool frt_index_is_deleted(FrtIndex *self, int doc_num)
-{
+bool frt_index_is_deleted(FrtIndex *self, int doc_num) {
     bool is_del;
     pthread_mutex_lock(&self->mutex);
     {
@@ -162,8 +146,7 @@ bool frt_index_is_deleted(FrtIndex *self, int doc_num)
 }
 
 static void index_del_doc_with_key_i(FrtIndex *self, FrtDocument *doc,
-                                            FrtHashSet *key)
-{
+                                            FrtHashSet *key) {
     FrtQuery *q;
     FrtTopDocs *td;
     FrtDocField *df;
@@ -199,8 +182,7 @@ static void index_del_doc_with_key_i(FrtIndex *self, FrtDocument *doc,
     frt_td_destroy(td);
 }
 
-static void index_add_doc_i(FrtIndex *self, FrtDocument *doc)
-{
+static void index_add_doc_i(FrtIndex *self, FrtDocument *doc) {
     if (self->key) {
         index_del_doc_with_key_i(self, doc, self->key);
     }
@@ -209,8 +191,7 @@ static void index_add_doc_i(FrtIndex *self, FrtDocument *doc)
     AUTOFLUSH_IW(self);
 }
 
-void frt_index_add_doc(FrtIndex *self, FrtDocument *doc)
-{
+void frt_index_add_doc(FrtIndex *self, FrtDocument *doc) {
     pthread_mutex_lock(&self->mutex);
     {
         index_add_doc_i(self, doc);
@@ -218,8 +199,7 @@ void frt_index_add_doc(FrtIndex *self, FrtDocument *doc)
     pthread_mutex_unlock(&self->mutex);
 }
 
-FrtQuery *frt_index_get_query(FrtIndex *self, char *qstr, rb_encoding *encoding)
-{
+FrtQuery *frt_index_get_query(FrtIndex *self, char *qstr, rb_encoding *encoding) {
     int i;
     FrtFieldInfos *fis;
     frt_ensure_searcher_open(self);
@@ -232,8 +212,7 @@ FrtQuery *frt_index_get_query(FrtIndex *self, char *qstr, rb_encoding *encoding)
 
 FrtTopDocs *frt_index_search_str(FrtIndex *self, char *qstr, int first_doc,
                           int num_docs, FrtFilter *filter, FrtSort *sort,
-                          FrtPostFilter *post_filter, rb_encoding *encoding)
-{
+                          FrtPostFilter *post_filter, rb_encoding *encoding) {
     FrtQuery *query;
     FrtTopDocs *td;
     query = frt_index_get_query(self, qstr, encoding); /* will ensure_searcher is open */
@@ -243,16 +222,14 @@ FrtTopDocs *frt_index_search_str(FrtIndex *self, char *qstr, int first_doc,
     return td;
 }
 
-FrtDocument *frt_index_get_doc(FrtIndex *self, int doc_num)
-{
+FrtDocument *frt_index_get_doc(FrtIndex *self, int doc_num) {
     FrtDocument *doc;
     frt_ensure_reader_open(self);
     doc = self->ir->get_doc(self->ir, doc_num);
     return doc;
 }
 
-FrtDocument *frt_index_get_doc_ts(FrtIndex *self, int doc_num)
-{
+FrtDocument *frt_index_get_doc_ts(FrtIndex *self, int doc_num) {
     FrtDocument *doc;
     pthread_mutex_lock(&self->mutex);
     {
@@ -278,13 +255,11 @@ FrtDocument *frt_index_get_doc_term(FrtIndex *self, ID field, const char *term)
     return doc;
 }
 
-FrtDocument *frt_index_get_doc_id(FrtIndex *self, const char *id)
-{
+FrtDocument *frt_index_get_doc_id(FrtIndex *self, const char *id) {
     return frt_index_get_doc_term(self, self->id_field, id);
 }
 
-void frt_index_delete(FrtIndex *self, int doc_num)
-{
+void frt_index_delete(FrtIndex *self, int doc_num) {
     pthread_mutex_lock(&self->mutex);
     {
         frt_ensure_reader_open(self);
@@ -316,20 +291,17 @@ void frt_index_delete_term(FrtIndex *self, ID field, const char *term) {
     pthread_mutex_unlock(&self->mutex);
 }
 
-void frt_index_delete_id(FrtIndex *self, const char *id)
-{
+void frt_index_delete_id(FrtIndex *self, const char *id) {
     frt_index_delete_term(self, self->id_field, id);
 }
 
-static void index_qdel_i(FrtSearcher *sea, int doc_num, float score, void *arg)
-{
+static void index_qdel_i(FrtSearcher *sea, int doc_num, float score, void *arg) {
     (void)score; (void)arg;
     frt_ir_delete_doc(((FrtIndexSearcher *)sea)->ir, doc_num);
 }
 
 void frt_index_delete_query(FrtIndex *self, FrtQuery *q, FrtFilter *f,
-                        FrtPostFilter *post_filter)
-{
+                        FrtPostFilter *post_filter) {
     pthread_mutex_lock(&self->mutex);
     {
         frt_ensure_searcher_open(self);
@@ -340,8 +312,7 @@ void frt_index_delete_query(FrtIndex *self, FrtQuery *q, FrtFilter *f,
 }
 
 void frt_index_delete_query_str(FrtIndex *self, char *qstr, FrtFilter *f,
-                            FrtPostFilter *post_filter, rb_encoding *encoding)
-{
+                            FrtPostFilter *post_filter, rb_encoding *encoding) {
     FrtQuery *q = frt_index_get_query(self, qstr, encoding);
     frt_index_delete_query(self, q, f, post_filter);
     frt_q_deref(q);

data/ext/isomorfeus_ferret_ext/frt_ind.h

@@ -32,7 +32,6 @@ typedef struct FrtIndex {
 extern FrtIndex *frt_index_new(FrtStore *store, FrtAnalyzer *analyzer, FrtHashSet *def_fields, bool create);
 extern void frt_index_destroy(FrtIndex *self);
 extern int frt_index_size(FrtIndex *self);
-extern void frt_index_optimize(FrtIndex *self);
 extern bool frt_index_is_deleted(FrtIndex *self, int doc_num);
 extern void frt_index_add_doc(FrtIndex *self, FrtDocument *doc);
 extern FrtTopDocs *frt_index_search_str(FrtIndex *self, char *query, int first_doc, int num_docs, FrtFilter *filter, FrtSort *sort, FrtPostFilter *post_filter, rb_encoding *encoding);