isomorfeus-ferret 0.17.1 → 0.17.2

data/ext/isomorfeus_ferret_ext/frt_index.c

@@ -1,4 +1,6 @@
 #include "frt_global.h"
+#include "frt_lazy_doc_field.h"
+#include "frt_lazy_doc.h"
 #include "frt_index.h"
 #include "frt_similarity.h"
 #include "frt_helper.h"
@@ -6,13 +8,6 @@
 #include <string.h>
 #include <limits.h>
 #include <ctype.h>
-#include "brotli_decode.h"
-#include "brotli_encode.h"
-#include "bzlib.h"
-#include "lz4frame.h"
-
-// #undef close
-// #undef read
 
 extern rb_encoding *utf8_encoding;
 extern void frt_micro_sleep(const int micro_seconds);
@@ -46,9 +41,6 @@ static char *ste_next(FrtTermEnum *te);
 #define FORMAT 15
 #define SEGMENTS_GEN_FILE_NAME "segments"
 #define MAX_EXT_LEN 10
-#define FRT_COMPRESSION_BUFFER_SIZE 16348
-#define FRT_BROTLI_COMPRESSION_LEVEL 4
-#define FRT_BZIP_COMPRESSION_LEVEL 9
 
 /* *** Must be three characters *** */
 static const char *INDEX_EXTENSIONS[] = {
@@ -590,29 +582,6 @@ static char *si_norm_file_name(FrtSegmentInfo *si, char *buf, int field_num)
 
 void frt_deleter_queue_file(FrtDeleter *dlr, const char *file_name);
 
-static void si_delete_files(FrtSegmentInfo *si, FrtFieldInfos *fis, FrtDeleter *dlr)
-{
-    int i;
-    char file_name[FRT_SEGMENT_NAME_MAX_LENGTH];
-    size_t seg_len = strlen(si->name);
-    char *ext;
-
-    for (i = si->norm_gens_size - 1; i >= 0; i--) {
-        if (0 <= si->norm_gens[i]) {
-            frt_deleter_queue_file(dlr, si_norm_file_name(si, file_name, fis->fields[i]->number));
-        }
-    }
-
-    memcpy(file_name, si->name, seg_len);
-    file_name[seg_len] = '.';
-    ext = file_name + seg_len + 1;
-
-    for (i = FRT_NELEMS(INDEX_EXTENSIONS) - 1; i >= 0; i--) {
-        memcpy(ext, INDEX_EXTENSIONS[i], 4);
-        frt_deleter_queue_file(dlr, file_name);
-    }
-}
-
 /****************************************************************************
  *
  * SegmentInfos
@@ -1044,355 +1013,6 @@ frt_u64 frt_sis_read_current_version(FrtStore *store)
     return fsf.ret.uint64;
 }
 
-/****************************************************************************
- *
- * LazyDocField
- *
- ****************************************************************************/
-
-static FrtLazyDocField *lazy_df_new(ID name, const int size, FrtCompressionType compression) {
-    FrtLazyDocField *self = FRT_ALLOC(FrtLazyDocField);
-    self->name = name;
-    self->size = size;
-    self->data = FRT_ALLOC_AND_ZERO_N(FrtLazyDocFieldData, size);
-    self->compression = compression;
-    self->decompressed = false;
-    self->loaded = false;
-    return self;
-}
-
-static void lazy_df_destroy(FrtLazyDocField *self) {
-    int i;
-    for (i = self->size - 1; i >= 0; i--) {
-        if (self->data[i].text) {
-            free(self->data[i].text);
-         }
-    }
-    free(self->data);
-    free(self);
-}
-
-static void comp_raise(void) {
-    FRT_RAISE(EXCEPTION, "Compression error");
-}
-
-static char *is_read_brotli_compressed_bytes(FrtInStream *is, int compressed_len, int *len) {
-    int buf_out_idx = 0;
-    int read_len;
-    frt_uchar buf_in[FRT_COMPRESSION_BUFFER_SIZE];
-    const frt_uchar *next_in;
-    size_t available_in;
-    frt_uchar *buf_out = NULL;
-    frt_uchar *next_out;
-    size_t available_out;
-
-    BrotliDecoderState *b_state = BrotliDecoderCreateInstance(NULL, NULL, NULL);
-    BrotliDecoderResult b_result = BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT;
-    if (!b_state) { comp_raise(); return NULL; }
-
-    do {
-        read_len = (compressed_len > FRT_COMPRESSION_BUFFER_SIZE) ? FRT_COMPRESSION_BUFFER_SIZE : compressed_len;
-        frt_is_read_bytes(is, buf_in, read_len);
-        compressed_len -= read_len;
-        available_in = read_len;
-        next_in = buf_in;
-        available_out = FRT_COMPRESSION_BUFFER_SIZE;
-        do {
-            FRT_REALLOC_N(buf_out, frt_uchar, buf_out_idx + FRT_COMPRESSION_BUFFER_SIZE);
-            next_out = buf_out + buf_out_idx;
-            b_result = BrotliDecoderDecompressStream(b_state,
-                &available_in, &next_in,
-                &available_out, &next_out, NULL);
-            if (b_result == BROTLI_DECODER_RESULT_ERROR) { comp_raise(); return NULL; }
-            buf_out_idx += FRT_COMPRESSION_BUFFER_SIZE - available_out;
-        } while (b_result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT);
-    } while (b_result != BROTLI_DECODER_RESULT_SUCCESS && compressed_len > 0);
-
-    BrotliDecoderDestroyInstance(b_state);
-
-    FRT_REALLOC_N(buf_out, frt_uchar, buf_out_idx + 1);
-    buf_out[buf_out_idx] = '\0';
-    *len = buf_out_idx;
-    return (char *)buf_out;
-}
-
-static void zraise(int ret) {
-    switch (ret) {
-    case BZ_IO_ERROR:
-        if (ferror(stdin))
-            FRT_RAISE(FRT_IO_ERROR, "bzlib: error reading stdin");
-        if (ferror(stdout))
-            FRT_RAISE(FRT_IO_ERROR, "bzlib: error writing stdout");
-        break;
-    case BZ_CONFIG_ERROR:
-        FRT_RAISE(FRT_IO_ERROR, "bzlib: system configuration error");
-        break;
-    case BZ_SEQUENCE_ERROR: /* shouldn't occur if code is correct */
-        FRT_RAISE(FRT_IO_ERROR, "bzlib: !!BUG!! sequence error");
-        break;
-    case BZ_PARAM_ERROR:    /* shouldn't occur if code is correct */
-        FRT_RAISE(FRT_IO_ERROR, "bzlib: !!BUG!! parameter error");
-        break;
-    case BZ_MEM_ERROR:
-        FRT_RAISE(FRT_IO_ERROR, "bzlib: memory error");
-        break;
-    case BZ_DATA_ERROR:
-        FRT_RAISE(FRT_IO_ERROR, "bzlib: data integrity check error");
-        break;
-    case BZ_DATA_ERROR_MAGIC:
-        FRT_RAISE(FRT_IO_ERROR, "bzlib: data integrity check - non-matching magic");
-        break;
-    case BZ_UNEXPECTED_EOF:
-        FRT_RAISE(FRT_IO_ERROR, "bzlib: unexpected end-of-file");
-        break;
-    case BZ_OUTBUFF_FULL:
-        FRT_RAISE(FRT_IO_ERROR, "bzlib: output buffer full");
-        break;
-    default:
-        FRT_RAISE(FRT_EXCEPTION, "bzlib: unknown error");
-    }
-}
-
-static char *is_read_bz2_compressed_bytes(FrtInStream *is, int compressed_len, int *len) {
-    int buf_out_idx = 0, ret, read_len;
-    char *buf_out = NULL;
-    char buf_in[FRT_COMPRESSION_BUFFER_SIZE];
-    bz_stream zstrm;
-    zstrm.bzalloc = NULL;
-    zstrm.bzfree  = NULL;
-    zstrm.opaque  = NULL;
-    zstrm.next_in = NULL;
-    zstrm.avail_in = 0;
-    if ((ret = BZ2_bzDecompressInit(&zstrm, 0, 0)) != BZ_OK) zraise(ret);
-
-    do {
-        read_len = (compressed_len > FRT_COMPRESSION_BUFFER_SIZE) ? FRT_COMPRESSION_BUFFER_SIZE : compressed_len;
-        frt_is_read_bytes(is, (frt_uchar *)buf_in, read_len);
-        compressed_len -= read_len;
-        zstrm.avail_in = read_len;
-        zstrm.next_in = buf_in;
-        zstrm.avail_out = FRT_COMPRESSION_BUFFER_SIZE;
-
-        do {
-            REALLOC_N(buf_out, char, buf_out_idx + FRT_COMPRESSION_BUFFER_SIZE);
-            zstrm.next_out = buf_out + buf_out_idx;
-            ret = BZ2_bzDecompress(&zstrm);
-            assert(ret != BZ_SEQUENCE_ERROR);  /* state not clobbered */
-            if (ret != BZ_OK && ret != BZ_STREAM_END) {
-                (void)BZ2_bzDecompressEnd(&zstrm);
-                zraise(ret);
-            }
-            buf_out_idx += FRT_COMPRESSION_BUFFER_SIZE - zstrm.avail_out;
-        } while (zstrm.avail_out == 0);
-    } while (ret != BZ_STREAM_END && compressed_len != 0);
-
-    (void)BZ2_bzDecompressEnd(&zstrm);
-
-    FRT_REALLOC_N(buf_out, char, buf_out_idx + 1);
-    buf_out[buf_out_idx] = '\0';
-
-    *len = buf_out_idx;
-    return (char *)buf_out;
-}
-
-static char *is_read_lz4_compressed_bytes(FrtInStream *is, int compressed_len, int *length) {
-    frt_uchar buf_in[FRT_COMPRESSION_BUFFER_SIZE];
-    char *buf_out = NULL;
-    int dc_length = 0;
-    LZ4F_dctx *dctx;
-    LZ4F_frameInfo_t frame_info;
-    LZ4F_errorCode_t dctx_status = LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION);
-    if (LZ4F_isError(dctx_status)) { *length = -1; return NULL; }
-
-    /* header and buffer */
-    int read_length = (compressed_len > FRT_COMPRESSION_BUFFER_SIZE) ? FRT_COMPRESSION_BUFFER_SIZE : compressed_len;
-    frt_is_read_bytes(is, buf_in, read_length);
-    compressed_len -= read_length;
-
-    size_t consumed_size = read_length;
-    size_t res = LZ4F_getFrameInfo(dctx, &frame_info, buf_in, &consumed_size);
-    if (LZ4F_isError(res)) { *length = -1; return NULL; }
-    size_t buf_out_length;
-    switch(frame_info.blockSizeID) {
-        case LZ4F_default:
-        case LZ4F_max64KB:
-            buf_out_length = 1 << 16;
-            break;
-        case LZ4F_max256KB:
-            buf_out_length = 1 << 18;
-            break;
-        case LZ4F_max1MB:
-            buf_out_length = 1 << 20;
-            break;
-        case LZ4F_max4MB:
-            buf_out_length = 1 << 22;
-            break;
-        default:
-            buf_out_length = 0;
-    }
-
-    res = 1;
-    int first_chunk = 1;
-
-    /* decompress data */
-    while (res != 0) {
-        if (!first_chunk) {
-            read_length = (compressed_len > FRT_COMPRESSION_BUFFER_SIZE) ? FRT_COMPRESSION_BUFFER_SIZE : compressed_len;
-            frt_is_read_bytes(is, buf_in, read_length);
-            compressed_len -= read_length;
-            consumed_size = 0;
-        }
-        first_chunk = 0;
-
-        char *src = (char *)(buf_in + consumed_size);
-        char *src_end = (char *)buf_in + read_length;
-
-        while (src < src_end && res != 0){
-            size_t dest_length = buf_out_length;
-            size_t consumed_size = read_length;
-            FRT_REALLOC_N(buf_out, char, dc_length + buf_out_length);
-            res = LZ4F_decompress(dctx, buf_out + dc_length, &dest_length, src, &consumed_size, NULL);
-            if (LZ4F_isError(res)) { *length = -1; return NULL; }
-            dc_length += dest_length;
-            src = src + consumed_size;
-        }
-    }
-
-    /* finish up */
-    LZ4F_freeDecompressionContext(dctx);
-
-    FRT_REALLOC_N(buf_out, char, dc_length + 1);
-    buf_out[dc_length] = '\0';
-
-    *length = dc_length;
-    return buf_out;
-}
-
-static char *is_read_compressed_bytes(FrtInStream *is, int compressed_len, int *len, FrtCompressionType compression) {
-    switch (compression) {
-        case FRT_COMPRESSION_BROTLI:
-            return is_read_brotli_compressed_bytes(is, compressed_len, len);
-        case FRT_COMPRESSION_BZ2:
-            return is_read_bz2_compressed_bytes(is, compressed_len, len);
-        case FRT_COMPRESSION_LZ4:
-            return is_read_lz4_compressed_bytes(is, compressed_len, len);
-        default:
-            return NULL;
-    }
-}
-
-char *frt_lazy_df_get_data(FrtLazyDocField *self, int i) {
-    char *text = NULL;
-    if (i < self->size && i >= 0) {
-        text = self->data[i].text;
-        if (NULL == text) {
-            const int read_len = self->data[i].length + 1;
-            frt_is_seek(self->doc->fields_in, self->data[i].start);
-            if (self->data[i].compression != FRT_COMPRESSION_NONE) {
-                self->data[i].text = text = is_read_compressed_bytes(self->doc->fields_in, read_len, &(self->data[i].length), self->data[i].compression);
-            } else {
-                self->data[i].text = text = FRT_ALLOC_N(char, read_len);
-                frt_is_read_bytes(self->doc->fields_in, (frt_uchar *)text, read_len);
-                text[read_len - 1] = '\0';
-            }
-            self->loaded = true;
-        }
-    }
-
-    return text;
-}
-
-void frt_lazy_df_get_bytes(FrtLazyDocField *self, char *buf, int start, int len) {
-    if (self->compression != FRT_COMPRESSION_NONE && !self->decompressed) {
-        int i;
-        self->len = 0;
-        for (i = self->size-1; i >= 0; i--) {
-            (void)frt_lazy_df_get_data(self, i);
-            self->len += self->data[i].length + 1;
-        }
-        self->len--; /* each field separated by ' ' but no need to add to end */
-        self->decompressed = true;
-    }
-    if (start < 0 || start >= self->len) {
-        FRT_RAISE(FRT_IO_ERROR, "start out of range in LazyDocField#get_bytes. %d "
-              "is not between 0 and %d", start, self->len);
-    }
-    if (len <= 0) {
-        FRT_RAISE(FRT_IO_ERROR, "len = %d, but should be greater than 0", len);
-    }
-    if (start + len > self->len) {
-        FRT_RAISE(FRT_IO_ERROR, "Tried to read past end of field. Field is only %d "
-              "bytes long but tried to read to %d", self->len, start + len);
-    }
-    if (self->compression != FRT_COMPRESSION_NONE) {
-        int cur_start = 0, buf_start = 0, cur_end, i, copy_start, copy_len;
-        for (i = 0; i < self->size; i++) {
-            cur_end = cur_start + self->data[i].length;
-            if (start < cur_end) {
-                copy_start = start > cur_start ? start - cur_start : 0;
-                copy_len = cur_end - cur_start - copy_start;
-                if (copy_len >= len) {
-                    copy_len = len;
-                    len = 0;
-                }
-                else {
-                    len -= copy_len;
-                }
-                memcpy(buf + buf_start,
-                       self->data[i].text + copy_start,
-                       copy_len);
-                buf_start += copy_len;
-                if (len > 0) {
-                    buf[buf_start++] = ' ';
-                    len--;
-                }
-                if (len == 0) break;
-            }
-            cur_start = cur_end + 1;
-        }
-    } else {
-        frt_is_seek(self->doc->fields_in, self->data[0].start + start);
-        frt_is_read_bytes(self->doc->fields_in, (frt_uchar *)buf, len);
-    }
-}
-
-/****************************************************************************
- *
- * LazyDoc
- *
- ****************************************************************************/
-
-static FrtLazyDoc *lazy_doc_new(int size, FrtInStream *fdt_in)
-{
-    FrtLazyDoc *self = FRT_ALLOC(FrtLazyDoc);
-    self->field_dictionary = frt_h_new_ptr((frt_free_ft)&lazy_df_destroy);
-    self->size = size;
-    self->fields = FRT_ALLOC_AND_ZERO_N(FrtLazyDocField *, size);
-    self->fields_in = frt_is_clone(fdt_in);
-    self->loaded = false;
-    return self;
-}
-
-void frt_lazy_doc_close(FrtLazyDoc *self)
-{
-    frt_h_destroy(self->field_dictionary);
-    frt_is_close(self->fields_in);
-    free(self->fields);
-    free(self);
-}
-
-static void lazy_doc_add_field(FrtLazyDoc *self, FrtLazyDocField *lazy_df, int i)
-{
-    self->fields[i] = lazy_df;
-
-    frt_h_set(self->field_dictionary, (void *)lazy_df->name, lazy_df);
-    lazy_df->doc = self;
-}
-
-FrtLazyDocField *frt_lazy_doc_get(FrtLazyDoc *self, ID field) {
-    return (FrtLazyDocField *)frt_h_get(self->field_dictionary, (void *)field);
-}
-
 /****************************************************************************
  * FrtFieldsReader
  ****************************************************************************/
@@ -1457,7 +1077,7 @@ static void frt_fr_read_compressed_fields(FrtFieldsReader *fr, FrtDocField *df,
 
     for (i = 0; i < df_size; i++) {
         const int compressed_len = df->lengths[i] + 1;
-        df->data[i] = is_read_compressed_bytes(fdt_in, compressed_len, &(df->lengths[i]), compression);
+        df->data[i] = frt_is_read_compressed_bytes(fdt_in, compressed_len, &(df->lengths[i]), compression);
     }
 }
 
@@ -1522,11 +1142,11 @@ FrtLazyDoc *frt_fr_get_lazy_doc(FrtFieldsReader *fr, int doc_num)
     frt_is_seek(fdt_in, pos);
     stored_cnt = frt_is_read_vint(fdt_in);
 
-    lazy_doc = lazy_doc_new(stored_cnt, fdt_in);
+    lazy_doc = frt_lazy_doc_new(stored_cnt, fdt_in);
     for (i = 0; i < stored_cnt; i++) {
         FrtFieldInfo *fi = fr->fis->fields[frt_is_read_vint(fdt_in)];
         const int df_size = frt_is_read_vint(fdt_in);
-        FrtLazyDocField *lazy_df = lazy_df_new(fi->name, df_size, frt_fi_get_compression(fi));
+        FrtLazyDocField *lazy_df = frt_lazy_df_new(fi->name, df_size, frt_fi_get_compression(fi));
         const int field_start = start;
         /* get the starts relative positions this time around */
 
@@ -1538,7 +1158,7 @@ FrtLazyDoc *frt_fr_get_lazy_doc(FrtFieldsReader *fr, int doc_num)
         }
 
         lazy_df->len = start - field_start - 1;
-        lazy_doc_add_field(lazy_doc, lazy_df, i);
+        frt_lazy_doc_add_field(lazy_doc, lazy_df, i);
     }
     /* correct the starts to their correct absolute positions */
     const frt_off_t abs_start = frt_is_pos(fdt_in);
@@ -1720,145 +1340,6 @@ void frt_fw_close(FrtFieldsWriter *fw) {
     free(fw);
 }
 
-static int frt_os_write_brotli_compressed_bytes(FrtOutStream* out_stream, frt_uchar *data, int length) {
-    size_t compressed_length = 0;
-    const frt_uchar *next_in = data;
-    size_t available_in = length;
-    size_t available_out;
-    frt_uchar compression_buffer[FRT_COMPRESSION_BUFFER_SIZE];
-    frt_uchar *next_out;
-    BrotliEncoderState *b_state = BrotliEncoderCreateInstance(NULL, NULL, NULL);
-    if (!b_state) { comp_raise(); return -1; }
-
-    BrotliEncoderSetParameter(b_state, BROTLI_PARAM_QUALITY, FRT_BROTLI_COMPRESSION_LEVEL);
-
-    do {
-        available_out = FRT_COMPRESSION_BUFFER_SIZE;
-        next_out = compression_buffer;
-        if (!BrotliEncoderCompressStream(b_state, BROTLI_OPERATION_FINISH,
-            &available_in, &next_in,
-            &available_out, &next_out, &compressed_length)) {
-            BrotliEncoderDestroyInstance(b_state);
-            comp_raise();
-            return -1;
-        }
-        frt_os_write_bytes(out_stream, compression_buffer, FRT_COMPRESSION_BUFFER_SIZE - available_out);
-    } while (!BrotliEncoderIsFinished(b_state));
-
-    BrotliEncoderDestroyInstance(b_state);
-
-    return (int)compressed_length;
-}
-
-static int frt_os_write_bz2_compressed_bytes(FrtOutStream* out_stream, frt_uchar *data, int length) {
-    int ret, buf_size, compressed_len = 0;
-    char out_buffer[FRT_COMPRESSION_BUFFER_SIZE];
-    bz_stream zstrm;
-    zstrm.bzalloc = NULL;
-    zstrm.bzfree  = NULL;
-    zstrm.opaque = NULL;
-    if ((ret = BZ2_bzCompressInit(&zstrm, FRT_BZIP_COMPRESSION_LEVEL, 0, 0)) != BZ_OK) zraise(ret);
-
-    zstrm.avail_in = length;
-    zstrm.next_in = (char *)data;
-    zstrm.avail_out = FRT_COMPRESSION_BUFFER_SIZE;
-    zstrm.next_out = out_buffer;
-
-    do {
-        ret = BZ2_bzCompress(&zstrm, BZ_FINISH); /* no bad return value */
-        assert(ret != BZ_SEQUENCE_ERROR);        /* state not clobbered */
-        compressed_len += buf_size = FRT_COMPRESSION_BUFFER_SIZE - zstrm.avail_out;
-        frt_os_write_bytes(out_stream, (frt_uchar *)out_buffer, buf_size);
-    } while (zstrm.avail_out == 0);
-    assert(zstrm.avail_in == 0);       /* all input will be used */
-
-    (void)BZ2_bzCompressEnd(&zstrm);
-    return compressed_len;
-}
-
-static const LZ4F_preferences_t lz4_prefs = {
-    {
-        LZ4F_default,
-        LZ4F_blockLinked,
-        LZ4F_noContentChecksum,
-        LZ4F_frame,
-        0, /* unknown content size */
-        0, /* no dictID */
-        LZ4F_noBlockChecksum
-    },
-    0,
-    1,
-    1,
-    {0,0,0}
-};
-
-static int frt_os_write_lz4_compressed_bytes(FrtOutStream* out_stream, frt_uchar *data, int length) {
-    int compressed_length = 0;
-    int remaining_length = length;
-    size_t ccmp_length = 0;
-    LZ4F_compressionContext_t ctx;
-    size_t out_buf_length = LZ4F_compressBound(FRT_COMPRESSION_BUFFER_SIZE, &lz4_prefs);
-    frt_uchar *out_buf = frt_ecalloc(out_buf_length);
-
-    size_t ctx_creation = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION);
-    if (LZ4F_isError(ctx_creation)) {
-        compressed_length = -1;
-        goto finish;
-    }
-
-    /* create header */
-    ccmp_length = LZ4F_compressBegin(ctx, out_buf, out_buf_length, &lz4_prefs);
-    if (LZ4F_isError(ccmp_length)) {
-        compressed_length = -1;
-        goto finish;
-    }
-    compressed_length = ccmp_length;
-    frt_os_write_bytes(out_stream, out_buf, ccmp_length);
-
-    /* compress data */
-    do {
-        int read_length = (FRT_COMPRESSION_BUFFER_SIZE > remaining_length) ? remaining_length : FRT_COMPRESSION_BUFFER_SIZE;
-        ccmp_length = LZ4F_compressUpdate(ctx, out_buf, out_buf_length, data + (length - remaining_length), read_length, NULL);
-        if (LZ4F_isError(ccmp_length)) {
-            compressed_length = -1;
-            goto finish;
-        }
-        frt_os_write_bytes(out_stream, out_buf, ccmp_length);
-        compressed_length += ccmp_length;
-        remaining_length -= read_length;
-    } while (remaining_length > 0);
-
-    /* finish up */
-    ccmp_length = LZ4F_compressEnd(ctx, out_buf, out_buf_length, NULL);
-    if (LZ4F_isError(ccmp_length)) {
-        compressed_length = -1;
-        goto finish;
-    }
-
-    frt_os_write_bytes(out_stream, out_buf, ccmp_length);
-    compressed_length += ccmp_length;
-
-finish:
-    LZ4F_freeCompressionContext(ctx);
-    free(out_buf);
-
-    return compressed_length;
-}
-
-static int frt_os_write_compressed_bytes(FrtOutStream* out_stream, frt_uchar *data, int length, FrtCompressionType compression) {
-    switch (compression) {
-        case FRT_COMPRESSION_BROTLI:
-            return frt_os_write_brotli_compressed_bytes(out_stream, data, length);
-        case FRT_COMPRESSION_BZ2:
-            return frt_os_write_bz2_compressed_bytes(out_stream, data, length);
-        case FRT_COMPRESSION_LZ4:
-            return frt_os_write_lz4_compressed_bytes(out_stream, data, length);
-        default:
-            return -1;
-    }
-
-}
-
 void frt_fw_add_doc(FrtFieldsWriter *fw, FrtDocument *doc) {
     int i, j, stored_cnt = 0;
     FrtDocField *df;
@@ -5384,465 +4865,6 @@ void frt_dw_add_doc(FrtDocWriter *dw, FrtDocument *doc) {
  * IndexWriter
  *
  ****************************************************************************/
-/****************************************************************************
- * SegmentMergeInfo
- ****************************************************************************/
-
-typedef struct SegmentMergeInfo {
-    int base;
-    int max_doc;
-    int doc_cnt;
-    FrtSegmentInfo *si;
-    FrtStore *store;
-    FrtStore *orig_store;
-    FrtBitVector *deleted_docs;
-    FrtSegmentFieldIndex *sfi;
-    FrtTermEnum *te;
-    FrtTermDocEnum *tde;
-    char *term;
-    int *doc_map;
-    FrtInStream *frq_in;
-    FrtInStream *prx_in;
-} SegmentMergeInfo;
-
-static bool smi_lt(const SegmentMergeInfo *smi1, const SegmentMergeInfo *smi2)
-{
-    int cmpres = strcmp(smi1->term, smi2->term);
-    if (0 == cmpres) {
-        return smi1->base < smi2->base;
-    }
-    else {
-        return cmpres < 0;
-    }
-}
-
-static void smi_load_doc_map(SegmentMergeInfo *smi)
-{
-    FrtBitVector *deleted_docs = smi->deleted_docs;
-    const int max_doc = smi->max_doc;
-    int j = 0, i;
-
-    smi->doc_map = FRT_ALLOC_N(int, max_doc);
-    for (i = 0; i < max_doc; i++) {
-        if (frt_bv_get(deleted_docs, i)) {
-            smi->doc_map[i] = -1;
-        }
-        else {
-            smi->doc_map[i] = j++;
-        }
-    }
-    smi->doc_cnt = j;
-}
-
-static SegmentMergeInfo *smi_new(int base, FrtStore *store, FrtSegmentInfo *si)
-{
-    SegmentMergeInfo *smi = FRT_ALLOC_AND_ZERO(SegmentMergeInfo);
-    char file_name[FRT_SEGMENT_NAME_MAX_LENGTH];
-    char *segment = si->name;
-    smi->base = base;
-    smi->si = si;
-    smi->orig_store = smi->store = store;
-    FRT_REF(smi->orig_store);
-
-    sprintf(file_name, "%s.fdx", segment);
-    smi->doc_cnt = smi->max_doc
-        = smi->store->length(smi->store, file_name) / FIELDS_IDX_PTR_SIZE;
-
-    if (si->del_gen >= 0) {
-        frt_fn_for_generation(file_name, segment, "del", si->del_gen);
-        smi->deleted_docs = bv_read(store, file_name);
-        smi_load_doc_map(smi);
-    }
-    return smi;
-}
-
-static void smi_load_term_input(SegmentMergeInfo *smi)
-{
-    FrtStore *store = smi->store;
-    char *segment = smi->si->name;
-    char file_name[FRT_SEGMENT_NAME_MAX_LENGTH];
-    smi->sfi = frt_sfi_open(store, segment);
-    sprintf(file_name, "%s.tis", segment);
-    FrtInStream *is = store->open_input(store, file_name);
-    FRT_DEREF(is);
-    smi->te = TE(frt_ste_new(is, smi->sfi));
-    sprintf(file_name, "%s.frq", segment);
-    smi->frq_in = store->open_input(store, file_name);
-    sprintf(file_name, "%s.prx", segment);
-    smi->prx_in = store->open_input(store, file_name);
-    smi->tde = frt_stpe_new(NULL, smi->frq_in, smi->prx_in, smi->deleted_docs,
-                        STE(smi->te)->skip_interval);
-}
-
-static void smi_close_term_input(SegmentMergeInfo *smi)
-{
-    frt_ste_close(smi->te);
-    frt_sfi_close(smi->sfi);
-    stpe_close(smi->tde);
-    frt_is_close(smi->frq_in);
-    frt_is_close(smi->prx_in);
-}
-
-static void smi_destroy(SegmentMergeInfo *smi)
-{
-    if (smi->store != smi->orig_store) {
-        frt_store_close(smi->store);
-    }
-    frt_store_close(smi->orig_store);
-    if (smi->deleted_docs) {
-        frt_bv_destroy(smi->deleted_docs);
-        free(smi->doc_map);
-    }
-    free(smi);
-}
-
-static char *smi_next(SegmentMergeInfo *smi)
-{
-    return (smi->term = ste_next(smi->te));
-}
-
-/****************************************************************************
- * SegmentMerger
- ****************************************************************************/
-
-typedef struct SegmentMerger {
-    FrtTermInfo ti;
-    FrtStore *store;
-    FrtFieldInfos *fis;
-    FrtSegmentInfo *si;
-    SegmentMergeInfo **smis;
-    int seg_cnt;
-    int doc_cnt;
-    FrtConfig *config;
-    FrtTermInfosWriter *tiw;
-    char *term_buf;
-    int term_buf_ptr;
-    int term_buf_size;
-    FrtPriorityQueue *queue;
-    SkipBuffer *skip_buf;
-    FrtOutStream *frq_out;
-    FrtOutStream *prx_out;
-} SegmentMerger;
-
-static SegmentMerger *sm_create(FrtIndexWriter *iw, FrtSegmentInfo *si, FrtSegmentInfo **seg_infos, const int seg_cnt)
-{
-    int i;
-    SegmentMerger *sm = FRT_ALLOC_AND_ZERO_N(SegmentMerger, seg_cnt);
-    sm->store = iw->store;
-    FRT_REF(sm->store);
-    sm->fis = iw->fis;
-    sm->si = si;
-    sm->doc_cnt = 0;
-    sm->smis = FRT_ALLOC_N(SegmentMergeInfo *, seg_cnt);
-    for (i = 0; i < seg_cnt; i++) {
-        sm->smis[i] = smi_new(sm->doc_cnt, seg_infos[i]->store, seg_infos[i]);
-        sm->doc_cnt += sm->smis[i]->doc_cnt;
-    }
-    sm->seg_cnt = seg_cnt;
-    sm->config = &iw->config;
-    return sm;
-}
-
-static void sm_destroy(SegmentMerger *sm)
-{
-    int i;
-    const int seg_cnt = sm->seg_cnt;
-    for (i = 0; i < seg_cnt; i++) {
-        smi_destroy(sm->smis[i]);
-    }
-    frt_store_close(sm->store);
-    free(sm->smis);
-    free(sm);
-}
-
-static void sm_merge_fields(SegmentMerger *sm)
-{
-    int i, j;
-    frt_off_t start, end = 0;
-    char file_name[FRT_SEGMENT_NAME_MAX_LENGTH];
-    FrtOutStream *fdt_out, *fdx_out;
-    FrtStore *store = sm->store;
-    const int seg_cnt = sm->seg_cnt;
-
-    sprintf(file_name, "%s.fdt", sm->si->name);
-    fdt_out = store->new_output(store, file_name);
-
-    sprintf(file_name, "%s.fdx", sm->si->name);
-    fdx_out = store->new_output(store, file_name);
-
-    for (i = 0; i < seg_cnt; i++) {
-        SegmentMergeInfo *smi = sm->smis[i];
-        const int max_doc = smi->max_doc;
-        FrtInStream *fdt_in, *fdx_in;
-        char *segment = smi->si->name;
-        store = smi->store;
-        sprintf(file_name, "%s.fdt", segment);
-        fdt_in = store->open_input(store, file_name);
-        sprintf(file_name, "%s.fdx", segment);
-        fdx_in = store->open_input(store, file_name);
-
-        if (max_doc > 0) {
-            end = (off_t)frt_is_read_u64(fdx_in);
-        }
-        for (j = 0; j < max_doc; j++) {
-            frt_u32 tv_idx_offset = frt_is_read_u32(fdx_in);
-            start = end;
-            if (j == max_doc - 1) {
-                end = frt_is_length(fdt_in);
-            }
-            else {
-                end = (off_t)frt_is_read_u64(fdx_in);
-            }
-            /* skip deleted docs */
-            if (!smi->deleted_docs || !frt_bv_get(smi->deleted_docs, j)) {
-                frt_os_write_u64(fdx_out, frt_os_pos(fdt_out));
-                frt_os_write_u32(fdx_out, tv_idx_offset);
-                frt_is_seek(fdt_in, start);
-                frt_is2os_copy_bytes(fdt_in, fdt_out, end - start);
-            }
-        }
-        frt_is_close(fdt_in);
-        frt_is_close(fdx_in);
-    }
-    frt_os_close(fdt_out);
-    frt_os_close(fdx_out);
-}
-
-static int sm_append_postings(SegmentMerger *sm, SegmentMergeInfo **matches,
-                              const int match_size)
-{
-    int i;
-    int last_doc = 0, base, doc, doc_code, freq;
-    int skip_interval = sm->config->skip_interval;
-    int *doc_map = NULL;
-    int df = 0;            /* number of docs w/ term */
-    FrtTermDocEnum *tde;
-    SegmentMergeInfo *smi;
-    SkipBuffer *skip_buf = sm->skip_buf;
-    skip_buf_reset(skip_buf);
-
-    for (i = 0; i < match_size; i++) {
-        smi = matches[i];
-        base = smi->base;
-        doc_map = smi->doc_map;
-        tde = smi->tde;
-        stpe_seek_ti(STDE(tde), &smi->te->curr_ti);
-
-        /* since we are using copy_bytes below to copy the proximities we use
-         * stde_next rather than stpe_next here */
-        while (stde_next(tde)) {
-            doc = stde_doc_num(tde);
-            if (NULL != doc_map) {
-                doc = doc_map[doc]; /* work around deletions */
-            }
-            doc += base;          /* convert to merged space */
-            assert(doc == 0 || doc > last_doc);
-
-            df++;
-            if (0 == (df % skip_interval)) {
-                skip_buf_add(skip_buf, last_doc);
-            }
-
-            doc_code = (doc - last_doc) << 1;    /* use low bit to flag freq=1 */
-            last_doc = doc;
-
-            freq = stde_freq(tde);
-            if (freq == 1) {
-                frt_os_write_vint(sm->frq_out, doc_code | 1); /* doc & freq=1 */
-            }
-            else {
-                frt_os_write_vint(sm->frq_out, doc_code); /* write doc */
-                frt_os_write_vint(sm->frq_out, freq);     /* write freqency in doc */
-            }
-
-            /* copy position deltas */
-            frt_is2os_copy_vints(STDE(tde)->prx_in, sm->prx_out, freq);
-        }
-    }
-    return df;
-}
-
-static char *sm_cache_term(SegmentMerger *sm, char *term, int term_len)
-{
-    term = (char *)memcpy(sm->term_buf + sm->term_buf_ptr, term, term_len + 1);
-    sm->term_buf_ptr += term_len + 1;
-    if (sm->term_buf_ptr > sm->term_buf_size) {
-        sm->term_buf_ptr = 0;
-    }
-    return term;
-}
-
-static void sm_merge_term_info(SegmentMerger *sm, SegmentMergeInfo **matches,
-                               int match_size)
-{
-    frt_off_t frq_ptr = frt_os_pos(sm->frq_out);
-    frt_off_t prx_ptr = frt_os_pos(sm->prx_out);
-
-    int df = sm_append_postings(sm, matches, match_size); /* append posting data */
-
-    frt_off_t skip_ptr = skip_buf_write(sm->skip_buf);
-
-    if (df > 0) {
-        /* add an entry to the dictionary with ptrs to prox and freq files */
-        SegmentMergeInfo *first_match = matches[0];
-        int term_len = first_match->te->curr_term_len;
-
-        frt_ti_set(sm->ti, df, frq_ptr, prx_ptr,
-               (skip_ptr - frq_ptr));
-        frt_tiw_add(sm->tiw, sm_cache_term(sm, first_match->term, term_len),
-                term_len, &sm->ti);
-    }
-}
-
-static void sm_merge_term_infos(SegmentMerger *sm)
-{
-    int i, j, match_size;
-    SegmentMergeInfo *smi, *top, **matches;
-    char *term;
-    const int seg_cnt = sm->seg_cnt;
-    const int fis_size = sm->fis->size;
-
-    matches = FRT_ALLOC_N(SegmentMergeInfo *, seg_cnt);
-
-    for (j = 0; j < seg_cnt; j++) {
-        smi_load_term_input(sm->smis[j]);
-    }
-
-    for (i = 0; i < fis_size; i++) {
-        frt_tiw_start_field(sm->tiw, i);
-        for (j = 0; j < seg_cnt; j++) {
-            smi = sm->smis[j];
-            ste_set_field(smi->te, i);
-            if (NULL != smi_next(smi)) {
-                frt_pq_push(sm->queue, smi); /* initialize @queue */
-            }
-        }
-        while (sm->queue->size > 0) {
-            match_size = 0;     /* pop matching terms */
-            matches[0] = (SegmentMergeInfo *)frt_pq_pop(sm->queue);
-            match_size++;
-            term = matches[0]->term;
-            top = (SegmentMergeInfo *)frt_pq_top(sm->queue);
-            while ((NULL != top) && (0 == strcmp(term, top->term))) {
-                matches[match_size] = (SegmentMergeInfo *)frt_pq_pop(sm->queue);
-                match_size++;
-                top = (SegmentMergeInfo *)frt_pq_top(sm->queue);
-            }
-
-            sm_merge_term_info(sm, matches, match_size);/* add new FrtTermInfo */
-
-            while (match_size > 0) {
-                match_size--;
-                smi = matches[match_size];
-                if (NULL != smi_next(smi)) {
-                    frt_pq_push(sm->queue, smi);   /* restore queue */
-                }
-            }
-        }
-    }
-    free(matches);
-    for (j = 0; j < seg_cnt; j++) {
-        smi_close_term_input(sm->smis[j]);
-    }
-}
-
-static void sm_merge_terms(SegmentMerger *sm)
-{
-    char file_name[FRT_SEGMENT_NAME_MAX_LENGTH];
-
-    sprintf(file_name, "%s.frq", sm->si->name);
-    sm->frq_out = sm->store->new_output(sm->store, file_name);
-    sprintf(file_name, "%s.prx", sm->si->name);
-    sm->prx_out = sm->store->new_output(sm->store, file_name);
-
-    sm->tiw = frt_tiw_open(sm->store, sm->si->name, sm->config->index_interval,
-                       sm->config->skip_interval);
-    sm->skip_buf = skip_buf_new(sm->frq_out, sm->prx_out);
-
-    /* terms_buf_ptr holds a buffer of terms since the FrtTermInfosWriter needs
-     * to keep the last index_interval terms so that it can compare the last
-     * term put in the index with the next one. So the size of the buffer must
-     * by index_interval + 2. */
-    sm->term_buf_ptr = 0;
-    sm->term_buf_size = (sm->config->index_interval + 1) * FRT_MAX_WORD_SIZE;
-    sm->term_buf = FRT_ALLOC_N(char, sm->term_buf_size + FRT_MAX_WORD_SIZE);
-
-    sm->queue = frt_pq_new(sm->seg_cnt, (frt_lt_ft)&smi_lt, NULL);
-
-    sm_merge_term_infos(sm);
-
-    frt_os_close(sm->frq_out);
-    frt_os_close(sm->prx_out);
-    frt_tiw_close(sm->tiw);
-    frt_pq_destroy(sm->queue);
-    skip_buf_destroy(sm->skip_buf);
-    free(sm->term_buf);
-}
-
-static void sm_merge_norms(SegmentMerger *sm)
-{
-    FrtSegmentInfo *si;
-    int i, j, k;
-    FrtStore *store;
-    frt_uchar byte;
-    FrtFieldInfo *fi;
-    FrtOutStream *os;
-    FrtInStream *is;
-    char file_name[FRT_SEGMENT_NAME_MAX_LENGTH];
-    SegmentMergeInfo *smi;
-    const int seg_cnt = sm->seg_cnt;
-    for (i = sm->fis->size - 1; i >= 0; i--) {
-        fi = sm->fis->fields[i];
-        if (bits_has_norms(fi->bits))  {
-            si = sm->si;
-            frt_si_advance_norm_gen(si, i);
-            si_norm_file_name(si, file_name, i);
-            os = sm->store->new_output(sm->store, file_name);
-            for (j = 0; j < seg_cnt; j++) {
-                smi = sm->smis[j];
-                si = smi->si;
-                if (si_norm_file_name(si, file_name, i)) {
-                    const int max_doc = smi->max_doc;
-                    FrtBitVector *deleted_docs =  smi->deleted_docs;
-                    store = smi->store;
-                    is = store->open_input(store, file_name);
-                    if (deleted_docs) {
-                        for (k = 0; k < max_doc; k++) {
-                            byte = frt_is_read_byte(is);
-                            if (!frt_bv_get(deleted_docs, k)) {
-                                frt_os_write_byte(os, byte);
-                            }
-                        }
-                    }
-                    else {
-                        frt_is2os_copy_bytes(is, os, max_doc);
-                    }
-                    frt_is_close(is);
-                }
-                else {
-                    const int doc_cnt = smi->doc_cnt;
-                    for (k = 0; k < doc_cnt; k++) {
-                        frt_os_write_byte(os, '\0');
-                    }
-                }
-            }
-            frt_os_close(os);
-        }
-    }
-}
-
-static int sm_merge(SegmentMerger *sm)
-{
-    sm_merge_fields(sm);
-    sm_merge_terms(sm);
-    sm_merge_norms(sm);
-    return sm->doc_cnt;
-}
-
-
-/****************************************************************************
- * IndexWriter
- ****************************************************************************/
 
 /* prepare an index ready for writing */
 void frt_index_create(FrtStore *store, FrtFieldInfos *fis)
@@ -5874,68 +4896,6 @@ int frt_iw_doc_count(FrtIndexWriter *iw)
     return doc_cnt;
 }
 
-static void iw_merge_segments(FrtIndexWriter *iw, const int min_seg, const int max_seg) {
-    int i;
-    FrtSegmentInfos *sis = iw->sis;
-    FrtSegmentInfo *si = frt_sis_new_segment(sis, 0, iw->store);
-
-    SegmentMerger *merger = sm_create(iw, si, &sis->segs[min_seg], max_seg - min_seg);
-
-    /* This is where all the action happens. */
-    si->doc_cnt = sm_merge(merger);
-
-    pthread_mutex_lock(&iw->store->mutex);
-    /* delete merged segments */
-    for (i = min_seg; i < max_seg; i++) {
-        si_delete_files(sis->segs[i], iw->fis, iw->deleter);
-    }
-
-    frt_sis_del_from_to(sis, min_seg, max_seg);
-
-    frt_sis_write(sis, iw->store, iw->deleter);
-    deleter_commit_pending_deletions(iw->deleter);
-
-    pthread_mutex_unlock(&iw->store->mutex);
-
-    sm_destroy(merger);
-}
-
-static void iw_merge_segments_from(FrtIndexWriter *iw, int min_segment)
-{
-    iw_merge_segments(iw, min_segment, iw->sis->size);
-}
-
-static void iw_maybe_merge_segments(FrtIndexWriter *iw)
-{
-    int target_merge_docs = iw->config.merge_factor;
-    int min_segment, merge_docs;
-    FrtSegmentInfo *si;
-
-    while (target_merge_docs > 0
-           && target_merge_docs <= iw->config.max_merge_docs) {
-        /* find segments smaller than current target size */
-        min_segment = iw->sis->size - 1;
-        merge_docs = 0;
-        while (min_segment >= 0) {
-            si = iw->sis->segs[min_segment];
-            if (si->doc_cnt >= target_merge_docs) {
-                break;
-            }
-            merge_docs += si->doc_cnt;
-            min_segment--;
-        }
-
-        if (merge_docs >= target_merge_docs) { /* found a merge to do */
-            iw_merge_segments_from(iw, min_segment + 1);
-        }
-        else if (min_segment <= 0) {
-            break;
-        }
-
-        target_merge_docs *= iw->config.merge_factor;
-    }
-}
-
 static void iw_flush_ram_segment(FrtIndexWriter *iw) {
     FrtSegmentInfos *sis = iw->sis;
     FrtSegmentInfo *si;
@@ -5947,7 +4907,6 @@ static void iw_flush_ram_segment(FrtIndexWriter *iw) {
     frt_sis_write(iw->sis, iw->store, iw->deleter);
     deleter_commit_pending_deletions(iw->deleter);
     pthread_mutex_unlock(&iw->store->mutex);
-    // iw_maybe_merge_segments(iw);
 }
 
 void frt_iw_add_doc(FrtIndexWriter *iw, FrtDocument *doc)
@@ -6051,26 +5010,6 @@ void frt_iw_delete_terms(FrtIndexWriter *iw, ID field, char **terms, const int t
     }
 }
 
-static void iw_optimize_i(FrtIndexWriter *iw)
-{
-    int min_segment;
-    iw_commit_i(iw);
-    // while (iw->sis->size > 1
-    //        || (iw->sis->size == 1
-    //            && (frt_si_has_deletions(iw->sis->segs[0])
-    //                || (iw->sis->segs[0]->store != iw->store)))) {
-    //     min_segment = iw->sis->size - iw->config.merge_factor;
-    //     iw_merge_segments_from(iw, min_segment < 0 ? 0 : min_segment);
-    // }
-}
-
-void frt_iw_optimize(FrtIndexWriter *iw)
-{
-    pthread_mutex_lock(&iw->mutex);
-    iw_optimize_i(iw);
-    pthread_mutex_unlock(&iw->mutex);
-}
-
 void frt_iw_close(FrtIndexWriter *iw)
 {
     pthread_mutex_lock(&iw->mutex);
@@ -6398,7 +5337,6 @@ static void iw_add_segments(FrtIndexWriter *iw, FrtIndexReader *ir) {
 void frt_iw_add_readers(FrtIndexWriter *iw, FrtIndexReader **readers, const int r_cnt) {
     int i;
     pthread_mutex_lock(&iw->mutex);
-    iw_optimize_i(iw);
 
     for (i = 0; i < r_cnt; i++) {
         iw_add_segments(iw, readers[i]);
@@ -6410,6 +5348,5 @@ void frt_iw_add_readers(FrtIndexWriter *iw, FrtIndexReader **readers, const int
     frt_sis_write(iw->sis, iw->store, iw->deleter);
     pthread_mutex_unlock(&iw->store->mutex);
 
-    iw_optimize_i(iw);
     pthread_mutex_unlock(&iw->mutex);
 }