isomorfeus-ferret 0.17.1 → 0.17.2

data/ext/isomorfeus_ferret_ext/frt_index.h

@@ -11,6 +11,7 @@
 #include "frt_similarity.h"
 #include "frt_bitvector.h"
 #include "frt_priorityqueue.h"
+#include "frt_lazy_doc.h"
 
 typedef struct FrtIndexReader FrtIndexReader;
 typedef struct FrtSegmentReader FrtSegmentReader;
@@ -495,48 +496,6 @@ extern int frt_tv_get_term_index(FrtTermVector *tv, const char *term);
 extern int frt_tv_scan_to_term_index(FrtTermVector *tv, const char *term);
 extern FrtTVTerm *frt_tv_get_tv_term(FrtTermVector *tv, const char *term);
 
-/****************************************************************************
- *
- * FrtLazyDoc
- *
- ****************************************************************************/
-
-/* * * FrtLazyDocField * * */
-typedef struct FrtLazyDocFieldData {
-    frt_off_t          start;
-    int                length;
-    rb_encoding        *encoding;
-    FrtCompressionType compression; /* as stored */
-    char               *text;
-} FrtLazyDocFieldData;
-
-typedef struct FrtLazyDoc FrtLazyDoc;
-typedef struct FrtLazyDocField {
-    ID                  name;
-    FrtLazyDocFieldData *data;
-    FrtLazyDoc          *doc;
-    int                 size; /* number of data elements */
-    int                 len;  /* length of data elements concatenated */
-    FrtCompressionType  compression; /* as configured */
-    bool                decompressed;
-    bool                loaded;
-} FrtLazyDocField;
-
-extern char *frt_lazy_df_get_data(FrtLazyDocField *self, int i);
-extern void frt_lazy_df_get_bytes(FrtLazyDocField *self, char *buf, int start, int len);
-
-/* * * FrtLazyDoc * * */
-struct FrtLazyDoc {
-    FrtHash         *field_dictionary;
-    int             size;
-    FrtLazyDocField **fields;
-    FrtInStream     *fields_in;
-    bool            loaded;
-};
-
-extern void frt_lazy_doc_close(FrtLazyDoc *self);
-extern FrtLazyDocField *frt_lazy_doc_get(FrtLazyDoc *self, ID field);
-
 /****************************************************************************
  *
  * FrtFieldsReader
@@ -810,7 +769,6 @@ extern void frt_iw_close(FrtIndexWriter *iw);
 extern void frt_iw_add_doc(FrtIndexWriter *iw, FrtDocument *doc);
 extern int frt_iw_doc_count(FrtIndexWriter *iw);
 extern void frt_iw_commit(FrtIndexWriter *iw);
-extern void frt_iw_optimize(FrtIndexWriter *iw);
 extern void frt_iw_add_readers(FrtIndexWriter *iw, FrtIndexReader **readers, const int r_cnt);
 
 /****************************************************************************

data/ext/isomorfeus_ferret_ext/frt_lazy_doc.c

@@ -0,0 +1,29 @@
+#include "frt_lazy_doc.h"
+
+FrtLazyDoc *frt_lazy_doc_new(int size, FrtInStream *fdt_in) {
+    FrtLazyDoc *self = FRT_ALLOC(FrtLazyDoc);
+    self->field_dictionary = frt_h_new_ptr((frt_free_ft)&frt_lazy_df_destroy);
+    self->size = size;
+    self->fields = FRT_ALLOC_AND_ZERO_N(FrtLazyDocField *, size);
+    self->fields_in = frt_is_clone(fdt_in);
+    self->loaded = false;
+    return self;
+}
+
+void frt_lazy_doc_close(FrtLazyDoc *self) {
+    frt_h_destroy(self->field_dictionary);
+    frt_is_close(self->fields_in);
+    free(self->fields);
+    free(self);
+}
+
+void frt_lazy_doc_add_field(FrtLazyDoc *self, FrtLazyDocField *lazy_df, int i) {
+    self->fields[i] = lazy_df;
+
+    frt_h_set(self->field_dictionary, (void *)lazy_df->name, lazy_df);
+    lazy_df->doc = self;
+}
+
+FrtLazyDocField *frt_lazy_doc_get(FrtLazyDoc *self, ID field) {
+    return (FrtLazyDocField *)frt_h_get(self->field_dictionary, (void *)field);
+}

data/ext/isomorfeus_ferret_ext/frt_lazy_doc.h

@@ -0,0 +1,19 @@
+#ifndef FRT_LAZY_DOC_H
+#define FRT_LAZY_DOC_H
+
+#include "frt_lazy_doc_field.h"
+
+typedef struct FrtLazyDoc {
+    FrtHash         *field_dictionary;
+    int             size;
+    FrtLazyDocField **fields;
+    FrtInStream     *fields_in;
+    bool            loaded;
+} FrtLazyDoc;
+
+extern FrtLazyDoc *frt_lazy_doc_new(int size, FrtInStream *fdt_in);
+extern void frt_lazy_doc_add_field(FrtLazyDoc *self, FrtLazyDocField *lazy_df, int i);
+extern void frt_lazy_doc_close(FrtLazyDoc *self);
+extern FrtLazyDocField *frt_lazy_doc_get(FrtLazyDoc *self, ID field);
+
+#endif

data/ext/isomorfeus_ferret_ext/frt_lazy_doc_field.c

@@ -0,0 +1,100 @@
+#include "frt_in_stream.h"
+#include "frt_lazy_doc_field.h"
+#include "frt_lazy_doc.h"
+
+FrtLazyDocField *frt_lazy_df_new(ID name, const int size, FrtCompressionType compression) {
+    FrtLazyDocField *self = FRT_ALLOC(FrtLazyDocField);
+    self->name = name;
+    self->size = size;
+    self->data = FRT_ALLOC_AND_ZERO_N(FrtLazyDocFieldData, size);
+    self->compression = compression;
+    self->decompressed = false;
+    self->loaded = false;
+    return self;
+}
+
+void frt_lazy_df_destroy(FrtLazyDocField *self) {
+    int i;
+    for (i = self->size - 1; i >= 0; i--) {
+        if (self->data[i].text) {
+            free(self->data[i].text);
+         }
+    }
+    free(self->data);
+    free(self);
+}
+
+char *frt_lazy_df_get_data(FrtLazyDocField *self, int i) {
+    char *text = NULL;
+    if (i < self->size && i >= 0) {
+        text = self->data[i].text;
+        if (NULL == text) {
+            const int read_len = self->data[i].length + 1;
+            frt_is_seek(self->doc->fields_in, self->data[i].start);
+            if (self->data[i].compression != FRT_COMPRESSION_NONE) {
+                self->data[i].text = text = frt_is_read_compressed_bytes(self->doc->fields_in, read_len, &(self->data[i].length), self->data[i].compression);
+            } else {
+                self->data[i].text = text = FRT_ALLOC_N(char, read_len);
+                frt_is_read_bytes(self->doc->fields_in, (frt_uchar *)text, read_len);
+                text[read_len - 1] = '\0';
+            }
+            self->loaded = true;
+        }
+    }
+
+    return text;
+}
+
+void frt_lazy_df_get_bytes(FrtLazyDocField *self, char *buf, int start, int len) {
+    if (self->compression != FRT_COMPRESSION_NONE && !self->decompressed) {
+        int i;
+        self->len = 0;
+        for (i = self->size-1; i >= 0; i--) {
+            (void)frt_lazy_df_get_data(self, i);
+            self->len += self->data[i].length + 1;
+        }
+        self->len--; /* each field separated by ' ' but no need to add to end */
+        self->decompressed = true;
+    }
+    if (start < 0 || start >= self->len) {
+        FRT_RAISE(FRT_IO_ERROR, "start out of range in LazyDocField#get_bytes. %d "
+              "is not between 0 and %d", start, self->len);
+    }
+    if (len <= 0) {
+        FRT_RAISE(FRT_IO_ERROR, "len = %d, but should be greater than 0", len);
+    }
+    if (start + len > self->len) {
+        FRT_RAISE(FRT_IO_ERROR, "Tried to read past end of field. Field is only %d "
+              "bytes long but tried to read to %d", self->len, start + len);
+    }
+    if (self->compression != FRT_COMPRESSION_NONE) {
+        int cur_start = 0, buf_start = 0, cur_end, i, copy_start, copy_len;
+        for (i = 0; i < self->size; i++) {
+            cur_end = cur_start + self->data[i].length;
+            if (start < cur_end) {
+                copy_start = start > cur_start ? start - cur_start : 0;
+                copy_len = cur_end - cur_start - copy_start;
+                if (copy_len >= len) {
+                    copy_len = len;
+                    len = 0;
+                }
+                else {
+                    len -= copy_len;
+                }
+                memcpy(buf + buf_start,
+                       self->data[i].text + copy_start,
+                       copy_len);
+                buf_start += copy_len;
+                if (len > 0) {
+                    buf[buf_start++] = ' ';
+                    len--;
+                }
+                if (len == 0) break;
+            }
+            cur_start = cur_end + 1;
+        }
+    } else {
+        frt_is_seek(self->doc->fields_in, self->data[0].start + start);
+        frt_is_read_bytes(self->doc->fields_in, (frt_uchar *)buf, len);
+    }
+}

data/ext/isomorfeus_ferret_ext/frt_lazy_doc_field.h

@@ -0,0 +1,33 @@
+#ifndef FRT_LAZY_DOC_FIELD_H
+#define FRT_LAZY_DOC_FIELD_H
+
+#include <ruby/encoding.h>
+#include "frt_hash.h"
+#include "frt_in_stream.h"
+
+typedef struct FrtLazyDocFieldData {
+    frt_off_t          start;
+    int                length;
+    rb_encoding        *encoding;
+    FrtCompressionType compression; /* as stored */
+    char               *text;
+} FrtLazyDocFieldData;
+
+typedef struct FrtLazyDoc FrtLazyDoc;
+typedef struct FrtLazyDocField {
+    ID                  name;
+    FrtLazyDocFieldData *data;
+    FrtLazyDoc          *doc;
+    int                 size; /* number of data elements */
+    int                 len;  /* length of data elements concatenated */
+    FrtCompressionType  compression; /* as configured */
+    bool                decompressed;
+    bool                loaded;
+} FrtLazyDocField;
+
+extern FrtLazyDocField *frt_lazy_df_new(ID name, const int size, FrtCompressionType compression);
+extern char *frt_lazy_df_get_data(FrtLazyDocField *self, int i);
+extern void frt_lazy_df_get_bytes(FrtLazyDocField *self, char *buf, int start, int len);
+extern void frt_lazy_df_destroy(FrtLazyDocField *self);
+
+#endif

data/ext/isomorfeus_ferret_ext/frt_out_stream.c

@@ -0,0 +1,346 @@
+#include "brotli_decode.h"
+#include "brotli_encode.h"
+#include "bzlib.h"
+#include "lz4frame.h"
+#include "frt_except.h"
+#include "frt_stream.h"
+#include "frt_out_stream.h"
+
+#define FRT_BROTLI_COMPRESSION_LEVEL 4
+#define FRT_BZIP_COMPRESSION_LEVEL 9
+
+/**
+ * Create a newly allocated and initialized OutStream object
+ *
+ * @return a newly allocated and initialized OutStream object
+ */
+FrtOutStream *frt_os_new(void) {
+    FrtOutStream *os = FRT_ALLOC(FrtOutStream);
+    os->buf.start = 0;
+    os->buf.pos = 0;
+    os->buf.len = 0;
+    return os;
+}
+
+/**
+ * Flush the countents of the FrtOutStream's buffers
+ *
+ * @param the OutStream to flush
+ */
+void frt_os_flush(FrtOutStream *os)
+{
+    os->m->flush_i(os, os->buf.buf, os->buf.pos);
+    os->buf.start += os->buf.pos;
+    os->buf.pos = 0;
+}
+
+void frt_os_close(FrtOutStream *os)
+{
+    frt_os_flush(os);
+    os->m->close_i(os);
+    free(os);
+}
+
+off_t frt_os_pos(FrtOutStream *os)
+{
+    return os->buf.start + os->buf.pos;
+}
+
+void frt_os_seek(FrtOutStream *os, frt_off_t new_pos)
+{
+    frt_os_flush(os);
+    os->buf.start = new_pos;
+    os->m->seek_i(os, new_pos);
+}
+
+/**
+ * Unsafe alternative to os_write_byte. Only use this method if you know there
+ * is no chance of buffer overflow.
+ */
+#define write_byte(os, b) os->buf.buf[os->buf.pos++] = (frt_uchar)b
+
+/**
+ * Write a single byte +b+ to the OutStream +os+
+ *
+ * @param os the OutStream to write to
+ * @param b  the byte to write
+ * @raise FRT_IO_ERROR if there is an IO error writing to the filesystem
+ */
+void frt_os_write_byte(FrtOutStream *os, frt_uchar b)
+{
+    if (os->buf.pos >= (FRT_BUFFER_SIZE - 1)) {
+        frt_os_flush(os);
+    }
+    write_byte(os, b);
+}
+
+void frt_os_write_bytes(FrtOutStream *os, const frt_uchar *buf, int len)
+{
+    if (len < (FRT_BUFFER_SIZE - os->buf.pos)) {
+        memcpy(os->buf.buf + os->buf.pos, buf, len);
+        os->buf.pos += len;
+    }
+    else {
+        frt_os_flush(os);
+        int pos = 0;
+        int size;
+        while (pos < len) {
+            if (len - pos < FRT_BUFFER_SIZE) {
+                size = len - pos;
+            }
+            else {
+                size = FRT_BUFFER_SIZE;
+            }
+            os->m->flush_i(os, buf + pos, size);
+            pos += size;
+            os->buf.start += size;
+        }
+    }
+}
+
+void frt_os_write_i32(FrtOutStream *os, frt_i32 num)
+{
+    frt_os_write_byte(os, (frt_uchar)((num >> 24) & 0xFF));
+    frt_os_write_byte(os, (frt_uchar)((num >> 16) & 0xFF));
+    frt_os_write_byte(os, (frt_uchar)((num >> 8) & 0xFF));
+    frt_os_write_byte(os, (frt_uchar)(num & 0xFF));
+}
+
+void frt_os_write_i64(FrtOutStream *os, frt_i64 num)
+{
+    frt_os_write_byte(os, (frt_uchar)((num >> 56) & 0xFF));
+    frt_os_write_byte(os, (frt_uchar)((num >> 48) & 0xFF));
+    frt_os_write_byte(os, (frt_uchar)((num >> 40) & 0xFF));
+    frt_os_write_byte(os, (frt_uchar)((num >> 32) & 0xFF));
+    frt_os_write_byte(os, (frt_uchar)((num >> 24) & 0xFF));
+    frt_os_write_byte(os, (frt_uchar)((num >> 16) & 0xFF));
+    frt_os_write_byte(os, (frt_uchar)((num >> 8) & 0xFF));
+    frt_os_write_byte(os, (frt_uchar)(num & 0xFF));
+}
+
+void frt_os_write_u32(FrtOutStream *os, frt_u32 num)
+{
+    frt_os_write_byte(os, (frt_uchar)((num >> 24) & 0xFF));
+    frt_os_write_byte(os, (frt_uchar)((num >> 16) & 0xFF));
+    frt_os_write_byte(os, (frt_uchar)((num >> 8) & 0xFF));
+    frt_os_write_byte(os, (frt_uchar)(num & 0xFF));
+}
+
+void frt_os_write_u64(FrtOutStream *os, frt_u64 num)
+{
+    frt_os_write_byte(os, (frt_uchar)((num >> 56) & 0xFF));
+    frt_os_write_byte(os, (frt_uchar)((num >> 48) & 0xFF));
+    frt_os_write_byte(os, (frt_uchar)((num >> 40) & 0xFF));
+    frt_os_write_byte(os, (frt_uchar)((num >> 32) & 0xFF));
+    frt_os_write_byte(os, (frt_uchar)((num >> 24) & 0xFF));
+    frt_os_write_byte(os, (frt_uchar)((num >> 16) & 0xFF));
+    frt_os_write_byte(os, (frt_uchar)((num >> 8) & 0xFF));
+    frt_os_write_byte(os, (frt_uchar)(num & 0xFF));
+}
+
+/* optimized to use an unchecked write if there is space */
+void frt_os_write_vint(FrtOutStream *os, register unsigned int num)
+{
+    if (os->buf.pos > VINT_END) {
+        while (num > 127) {
+            frt_os_write_byte(os, (frt_uchar)((num & 0x7f) | 0x80));
+            num >>= 7;
+        }
+        frt_os_write_byte(os, (frt_uchar)(num));
+    }
+    else {
+        while (num > 127) {
+            write_byte(os, (frt_uchar)((num & 0x7f) | 0x80));
+            num >>= 7;
+        }
+        write_byte(os, (frt_uchar)(num));
+    }
+}
+
+/* optimized to use an unchecked write if there is space */
+void frt_os_write_voff_t(FrtOutStream *os, register frt_off_t num)
+{
+    if (os->buf.pos > VINT_END) {
+        while (num > 127) {
+            frt_os_write_byte(os, (frt_uchar)((num & 0x7f) | 0x80));
+            num >>= 7;
+        }
+        frt_os_write_byte(os, (frt_uchar)num);
+    }
+    else {
+        while (num > 127) {
+            write_byte(os, (frt_uchar)((num & 0x7f) | 0x80));
+            num >>= 7;
+        }
+        write_byte(os, (frt_uchar)num);
+    }
+}
+
+/* optimized to use an unchecked write if there is space */
+void frt_os_write_vll(FrtOutStream *os, register frt_u64 num)
+{
+    if (os->buf.pos > VINT_END) {
+        while (num > 127) {
+            frt_os_write_byte(os, (frt_uchar)((num & 0x7f) | 0x80));
+            num >>= 7;
+        }
+        frt_os_write_byte(os, (frt_uchar)num);
+    }
+    else {
+        while (num > 127) {
+            write_byte(os, (frt_uchar)((num & 0x7f) | 0x80));
+            num >>= 7;
+        }
+        write_byte(os, (frt_uchar)num);
+    }
+}
+
+void frt_os_write_string_len(FrtOutStream *os, const char *str, int len)
+{
+    frt_os_write_vint(os, len);
+    frt_os_write_bytes(os, (frt_uchar *)str, len);
+}
+
+void frt_os_write_string(FrtOutStream *os, const char *str)
+{
+    frt_os_write_string_len(os, str, (int)strlen(str));
+}
+
+static int frt_os_write_brotli_compressed_bytes(FrtOutStream* out_stream, frt_uchar *data, int length) {
+    size_t compressed_length = 0;
+    const frt_uchar *next_in = data;
+    size_t available_in = length;
+    size_t available_out;
+    frt_uchar compression_buffer[FRT_COMPRESSION_BUFFER_SIZE];
+    frt_uchar *next_out;
+    BrotliEncoderState *b_state = BrotliEncoderCreateInstance(NULL, NULL, NULL);
+    if (!b_state) { frt_comp_raise(); return -1; }
+
+    BrotliEncoderSetParameter(b_state, BROTLI_PARAM_QUALITY, FRT_BROTLI_COMPRESSION_LEVEL);
+
+    do {
+        available_out = FRT_COMPRESSION_BUFFER_SIZE;
+        next_out = compression_buffer;
+        if (!BrotliEncoderCompressStream(b_state, BROTLI_OPERATION_FINISH,
+            &available_in, &next_in,
+            &available_out, &next_out, &compressed_length)) {
+            BrotliEncoderDestroyInstance(b_state);
+            frt_comp_raise();
+            return -1;
+        }
+        frt_os_write_bytes(out_stream, compression_buffer, FRT_COMPRESSION_BUFFER_SIZE - available_out);
+    } while (!BrotliEncoderIsFinished(b_state));
+
+    BrotliEncoderDestroyInstance(b_state);
+
+    return (int)compressed_length;
+}
+
+static int frt_os_write_bz2_compressed_bytes(FrtOutStream* out_stream, frt_uchar *data, int length) {
+    int ret, buf_size, compressed_len = 0;
+    char out_buffer[FRT_COMPRESSION_BUFFER_SIZE];
+    bz_stream zstrm;
+    zstrm.bzalloc = NULL;
+    zstrm.bzfree  = NULL;
+    zstrm.opaque = NULL;
+    if ((ret = BZ2_bzCompressInit(&zstrm, FRT_BZIP_COMPRESSION_LEVEL, 0, 0)) != BZ_OK) frt_zraise(ret);
+
+    zstrm.avail_in = length;
+    zstrm.next_in = (char *)data;
+    zstrm.avail_out = FRT_COMPRESSION_BUFFER_SIZE;
+    zstrm.next_out = out_buffer;
+
+    do {
+        ret = BZ2_bzCompress(&zstrm, BZ_FINISH); /* no bad return value */
+        assert(ret != BZ_SEQUENCE_ERROR);        /* state not clobbered */
+        compressed_len += buf_size = FRT_COMPRESSION_BUFFER_SIZE - zstrm.avail_out;
+        frt_os_write_bytes(out_stream, (frt_uchar *)out_buffer, buf_size);
+    } while (zstrm.avail_out == 0);
+    assert(zstrm.avail_in == 0);       /* all input will be used */
+
+    (void)BZ2_bzCompressEnd(&zstrm);
+    return compressed_len;
+}
+
+static const LZ4F_preferences_t lz4_prefs = {
+    {
+        LZ4F_default,
+        LZ4F_blockLinked,
+        LZ4F_noContentChecksum,
+        LZ4F_frame,
+        0, /* unknown content size */
+        0, /* no dictID */
+        LZ4F_noBlockChecksum
+    },
+    0,
+    1,
+    1,
+    {0,0,0}
+};
+
+static int frt_os_write_lz4_compressed_bytes(FrtOutStream* out_stream, frt_uchar *data, int length) {
+    int compressed_length = 0;
+    int remaining_length = length;
+    size_t ccmp_length = 0;
+    LZ4F_compressionContext_t ctx;
+    size_t out_buf_length = LZ4F_compressBound(FRT_COMPRESSION_BUFFER_SIZE, &lz4_prefs);
+    frt_uchar *out_buf = frt_ecalloc(out_buf_length);
+
+    size_t ctx_creation = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION);
+    if (LZ4F_isError(ctx_creation)) {
+        compressed_length = -1;
+        goto finish;
+    }
+
+    /* create header */
+    ccmp_length = LZ4F_compressBegin(ctx, out_buf, out_buf_length, &lz4_prefs);
+    if (LZ4F_isError(ccmp_length)) {
+        compressed_length = -1;
+        goto finish;
+    }
+    compressed_length = ccmp_length;
+    frt_os_write_bytes(out_stream, out_buf, ccmp_length);
+
+    /* compress data */
+    do {
+        int read_length = (FRT_COMPRESSION_BUFFER_SIZE > remaining_length) ? remaining_length : FRT_COMPRESSION_BUFFER_SIZE;
+        ccmp_length = LZ4F_compressUpdate(ctx, out_buf, out_buf_length, data + (length - remaining_length), read_length, NULL);
+        if (LZ4F_isError(ccmp_length)) {
+            compressed_length = -1;
+            goto finish;
+        }
+        frt_os_write_bytes(out_stream, out_buf, ccmp_length);
+        compressed_length += ccmp_length;
+        remaining_length -= read_length;
+    } while (remaining_length > 0);
+
+    /* finish up */
+    ccmp_length = LZ4F_compressEnd(ctx, out_buf, out_buf_length, NULL);
+    if (LZ4F_isError(ccmp_length)) {
+        compressed_length = -1;
+        goto finish;
+    }
+
+    frt_os_write_bytes(out_stream, out_buf, ccmp_length);
+    compressed_length += ccmp_length;
+
+finish:
+    LZ4F_freeCompressionContext(ctx);
+    free(out_buf);
+
+    return compressed_length;
+}
+
+int frt_os_write_compressed_bytes(FrtOutStream* out_stream, frt_uchar *data, int length, FrtCompressionType compression) {
+    switch (compression) {
+        case FRT_COMPRESSION_BROTLI:
+            return frt_os_write_brotli_compressed_bytes(out_stream, data, length);
+        case FRT_COMPRESSION_BZ2:
+            return frt_os_write_bz2_compressed_bytes(out_stream, data, length);
+        case FRT_COMPRESSION_LZ4:
+            return frt_os_write_lz4_compressed_bytes(out_stream, data, length);
+        default:
+            return -1;
+    }
+
+}