isomorfeus-ferret 0.17.1 → 0.17.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3e6d632f920215dd3c9a1b8833a255c2001102baac820a88b322308b023600b1
-  data.tar.gz: 571a40b2b445f52f3044b732eb9356d0e6f8dd9d7d32e40eaa50ef66cd92b054
+  metadata.gz: 9d39decdeae08fb0fa83f242e3162938e10a9a7549ba3efcc1f5fbb06238e695
+  data.tar.gz: 89679c3ca82432887764672840965ddcc7633ea64ee5e79c92fbd4a9615e6771
 SHA512:
-  metadata.gz: 8ac4670e9bb4bc9e3019593d82901d6aa7a4112996f6c48cefd0b98f87822daac4306331f196e3be1a46189689ed8b02a3787f5568e00071ba5706e43a7bb6b5
-  data.tar.gz: 86d921d0b200ae74b2b3d86867450d8ab02409e4aa56ddc3fa4f9ba3ae4cf336155202440accffb315996df4dfd89247743b951e4856037f736a36251ab7a724
+  metadata.gz: 84b855487d6c7c390be71b36398f38ef016459773f0bd0412b79953ed752acef3a8584a43c455721c145ca95e4f384bee070d20baa25b709ed641cbb8db9ba2d
+  data.tar.gz: a229b8f4d6e93ad9b98d8ea87bff6ae124cc75d789302609a2994149264dc1d28f28bde4a24e389b0ee28d935a6b42a7c8671ed2e75ea42db026bd01769ac817

data/ext/isomorfeus_ferret_ext/bm_store.c

@@ -1,4 +1,6 @@
 #include <string.h>
+#include "frt_in_stream.h"
+#include "frt_out_stream.h"
 #include "benchmark.h"
 #include "frt_config.h"
 #include "frt_store.h"

data/ext/isomorfeus_ferret_ext/frb_index.c

@@ -1258,40 +1258,6 @@ frb_iw_add_doc(VALUE self, VALUE rdoc) {
     return self;
 }
 
-/*
- *  call-seq:
- *     iw.optimize -> iw
- *
- *  Optimize the index for searching. This commits any unwritten data to the
- *  index and optimizes the index into a single segment to improve search
- *  performance. This is an expensive operation and should not be called too
- *  often. The best time to call this is at the end of a long batch indexing
- *  process. Note that calling the optimize method do not in any way effect
- *  indexing speed (except for the time taken to complete the optimization
- *  process).
- */
-static VALUE
-frb_iw_optimize(VALUE self) {
-    int ex_code = 0;
-    const char *msg = NULL;
-
-    FrtIndexWriter *iw = (FrtIndexWriter *)DATA_PTR(self);
-
-    FRT_TRY
-        frt_iw_optimize(iw);
-    FRT_XCATCHALL
-        ex_code = xcontext.excode;
-        msg = xcontext.msg;
-        FRT_HANDLED();
-    FRT_XENDTRY
-
-    if (ex_code && msg) {
-        frb_raise(ex_code, msg);
-    }
-
-    return self;
-}
-
 /*
  *  call-seq:
  *     iw.commit -> iw
@@ -2811,7 +2777,6 @@ void Init_IndexWriter(void) {
     rb_define_method(cIndexWriter, "close",        frb_iw_close, 0);
     rb_define_method(cIndexWriter, "add_document", frb_iw_add_doc, 1);
     rb_define_method(cIndexWriter, "<<",           frb_iw_add_doc, 1);
-    rb_define_method(cIndexWriter, "optimize",     frb_iw_optimize, 0);
     rb_define_method(cIndexWriter, "commit",       frb_iw_commit, 0);
     rb_define_method(cIndexWriter, "add_readers",  frb_iw_add_readers, 1);
     rb_define_method(cIndexWriter, "delete",       frb_iw_delete, 2);

data/ext/isomorfeus_ferret_ext/frt_except.c

@@ -1,4 +1,5 @@
 #include <stdarg.h>
+#include "bzlib.h"
 #include "frt_global.h"
 #include "frt_except.h"
 #include "frt_threading.h"
@@ -71,6 +72,47 @@ void frt_xraise(int excode, const char *const msg) {
     }
 }
 
+void frt_comp_raise(void) {
+    FRT_RAISE(EXCEPTION, "Compression error");
+}
+
+void frt_zraise(int ret) {
+    switch (ret) {
+    case BZ_IO_ERROR:
+        if (ferror(stdin))
+            FRT_RAISE(FRT_IO_ERROR, "bzlib: error reading stdin");
+        if (ferror(stdout))
+            FRT_RAISE(FRT_IO_ERROR, "bzlib: error writing stdout");
+        break;
+    case BZ_CONFIG_ERROR:
+        FRT_RAISE(FRT_IO_ERROR, "bzlib: system configuration error");
+        break;
+    case BZ_SEQUENCE_ERROR: /* shouldn't occur if code is correct */
+        FRT_RAISE(FRT_IO_ERROR, "bzlib: !!BUG!! sequence error");
+        break;
+    case BZ_PARAM_ERROR:    /* shouldn't occur if code is correct */
+        FRT_RAISE(FRT_IO_ERROR, "bzlib: !!BUG!! parameter error");
+        break;
+    case BZ_MEM_ERROR:
+        FRT_RAISE(FRT_IO_ERROR, "bzlib: memory error");
+        break;
+    case BZ_DATA_ERROR:
+        FRT_RAISE(FRT_IO_ERROR, "bzlib: data integrity check error");
+        break;
+    case BZ_DATA_ERROR_MAGIC:
+        FRT_RAISE(FRT_IO_ERROR, "bzlib: data integrity check - non-matching magic");
+        break;
+    case BZ_UNEXPECTED_EOF:
+        FRT_RAISE(FRT_IO_ERROR, "bzlib: unexpected end-of-file");
+        break;
+    case BZ_OUTBUFF_FULL:
+        FRT_RAISE(FRT_IO_ERROR, "bzlib: output buffer full");
+        break;
+    default:
+        FRT_RAISE(FRT_EXCEPTION, "bzlib: unknown error");
+    }
+}
+
 void frt_xpop_context(void) {
     frt_xcontext_t *top_cxt, *context;
     pthread_once(&exception_stack_key_once, *exception_stack_alloc);

data/ext/isomorfeus_ferret_ext/frt_except.h

@@ -178,6 +178,8 @@ typedef struct frt_xcontext_t
 extern void FRT_RAISE(int excode, const char *fmt, ...);
 #endif
 
+extern void frt_comp_raise(void);
+extern void frt_zraise(int ret);
 extern void frt_xraise(int excode, const char *const msg);
 extern void frt_xpush_context(frt_xcontext_t *context);
 extern void frt_xpop_context();

data/ext/isomorfeus_ferret_ext/frt_in_stream.c

@@ -0,0 +1,492 @@
+#include "brotli_decode.h"
+#include "brotli_encode.h"
+#include "bzlib.h"
+#include "lz4frame.h"
+#include "frt_except.h"
+#include "frt_in_stream.h"
+
+/**
+ * Create a newly allocated and initialized InStream
+ *
+ * @return a newly allocated and initialized InStream
+ */
+FrtInStream *frt_is_new(void) {
+    FrtInStream *is = FRT_ALLOC(FrtInStream);
+    is->f = FRT_ALLOC_AND_ZERO(FrtInStreamFile);
+    is->f->ref_cnt = 1;
+    is->buf.start = 0;
+    is->buf.pos = 0;
+    is->buf.len = 0;
+    is->ref_cnt = 1;
+    return is;
+}
+
+/**
+ * Refill the InStream's buffer from the store source (filesystem or memory).
+ *
+ * @param is the InStream to refill
+ * @raise FRT_IO_ERROR if there is a error reading from the filesystem
+ * @raise FRT_EOF_ERROR if there is an attempt to read past the end of the file
+ */
+static void frt_is_refill(FrtInStream *is)
+{
+    frt_off_t start = is->buf.start + is->buf.pos;
+    frt_off_t last = start + FRT_BUFFER_SIZE;
+    frt_off_t flen = is->m->length_i(is);
+
+    if (last > flen) {          /* don't read past EOF */
+        last = flen;
+    }
+
+    is->buf.len = last - start;
+    if (is->buf.len <= 0) {
+        FRT_RAISE(FRT_EOF_ERROR, "current pos = %"FRT_OFF_T_PFX"d, "
+              "file length = %"FRT_OFF_T_PFX"d", start, flen);
+    }
+
+    is->m->read_i(is, is->buf.buf, is->buf.len);
+
+    is->buf.start = start;
+    is->buf.pos = 0;
+}
+
+/**
+ * Unsafe alternative to frt_is_read_byte. Only use this method when you know
+ * there is no chance that you will read past the end of the InStream's
+ * buffer.
+ */
+#define read_byte(is) is->buf.buf[is->buf.pos++]
+
+/**
+ * Read a singly byte (unsigned char) from the InStream +is+.
+ *
+ * @param is the Instream to read from
+ * @return a single unsigned char read from the InStream +is+
+ * @raise FRT_IO_ERROR if there is a error reading from the filesystem
+ * @raise FRT_EOF_ERROR if there is an attempt to read past the end of the file
+ */
+frt_uchar frt_is_read_byte(FrtInStream *is)
+{
+    if (is->buf.pos >= is->buf.len) {
+        frt_is_refill(is);
+    }
+
+    return read_byte(is);
+}
+
+off_t frt_is_pos(FrtInStream *is)
+{
+    return is->buf.start + is->buf.pos;
+}
+
+frt_uchar *frt_is_read_bytes(FrtInStream *is, frt_uchar *buf, int len)
+{
+    int i;
+    frt_off_t start;
+
+    if ((is->buf.pos + len) < is->buf.len) {
+        for (i = 0; i < len; i++) {
+            buf[i] = read_byte(is);
+        }
+    }
+    else {                              /* read all-at-once */
+        start = frt_is_pos(is);
+        is->m->seek_i(is, start);
+        is->m->read_i(is, buf, len);
+
+        is->buf.start = start + len;    /* adjust stream variables */
+        is->buf.pos = 0;
+        is->buf.len = 0;                /* trigger refill on read */
+    }
+    return buf;
+}
+
+void frt_is_seek(FrtInStream *is, frt_off_t pos) {
+    if (pos >= is->buf.start && pos < (is->buf.start + is->buf.len)) {
+        is->buf.pos = pos - is->buf.start;  /* seek within buffer */
+    } else {
+        is->buf.start = pos;
+        is->buf.pos = 0;
+        is->buf.len = 0;                    /* trigger refill() on read() */
+        is->m->seek_i(is, pos);
+    }
+}
+
+void frt_is_close(FrtInStream *is) {
+    if (is->ref_cnt == 0) {
+        FRT_RAISE(FRT_STATE_ERROR, "is ref_cnt to low\n");
+    }
+
+    if (FRT_DEREF(is) == 0) {
+        if (FRT_DEREF(is->f) == 0) {
+            is->m->close_i(is);
+            free(is->f);
+        }
+        free(is);
+    }
+}
+
+FrtInStream *frt_is_clone(FrtInStream *is)
+{
+    if (!(is->f))
+        return NULL;
+    FrtInStream *new_is = FRT_ALLOC(FrtInStream);
+    memcpy(new_is, is, sizeof(FrtInStream));
+    new_is->ref_cnt = 1;
+    FRT_REF(new_is->f);
+    return new_is;
+}
+
+frt_i32 frt_is_read_i32(FrtInStream *is)
+{
+    return ((frt_i32)frt_is_read_byte(is) << 24) |
+        ((frt_i32)frt_is_read_byte(is) << 16) |
+        ((frt_i32)frt_is_read_byte(is) << 8) |
+        ((frt_i32)frt_is_read_byte(is));
+}
+
+
+frt_i64 frt_is_read_i64(FrtInStream *is)
+{
+    return ((frt_i64)frt_is_read_byte(is) << 56) |
+        ((frt_i64)frt_is_read_byte(is) << 48) |
+        ((frt_i64)frt_is_read_byte(is) << 40) |
+        ((frt_i64)frt_is_read_byte(is) << 32) |
+        ((frt_i64)frt_is_read_byte(is) << 24) |
+        ((frt_i64)frt_is_read_byte(is) << 16) |
+        ((frt_i64)frt_is_read_byte(is) << 8) |
+        ((frt_i64)frt_is_read_byte(is));
+}
+
+
+frt_u32 frt_is_read_u32(FrtInStream *is)
+{
+    return ((frt_u32)frt_is_read_byte(is) << 24) |
+        ((frt_u32)frt_is_read_byte(is) << 16) |
+        ((frt_u32)frt_is_read_byte(is) << 8) |
+        ((frt_u32)frt_is_read_byte(is));
+}
+
+frt_u64 frt_is_read_u64(FrtInStream *is)
+{
+    return ((frt_u64)frt_is_read_byte(is) << 56) |
+        ((frt_u64)frt_is_read_byte(is) << 48) |
+        ((frt_u64)frt_is_read_byte(is) << 40) |
+        ((frt_u64)frt_is_read_byte(is) << 32) |
+        ((frt_u64)frt_is_read_byte(is) << 24) |
+        ((frt_u64)frt_is_read_byte(is) << 16) |
+        ((frt_u64)frt_is_read_byte(is) << 8) |
+        ((frt_u64)frt_is_read_byte(is));
+}
+
+
+/* optimized to use unchecked read_byte if there is definitely space */
+unsigned int frt_is_read_vint(FrtInStream *is)
+{
+    register unsigned int res, b;
+    register int shift = 7;
+
+    if (is->buf.pos > (is->buf.len - VINT_MAX_LEN)) {
+        b = frt_is_read_byte(is);
+        res = b & 0x7F;                 /* 0x7F = 0b01111111 */
+
+        while ((b & 0x80) != 0) {       /* 0x80 = 0b10000000 */
+            b = frt_is_read_byte(is);
+            res |= (b & 0x7F) << shift;
+            shift += 7;
+        }
+    }
+    else {                              /* unchecked optimization */
+        b = read_byte(is);
+        res = b & 0x7F;                 /* 0x7F = 0b01111111 */
+
+        while ((b & 0x80) != 0) {       /* 0x80 = 0b10000000 */
+            b = read_byte(is);
+            res |= (b & 0x7F) << shift;
+            shift += 7;
+        }
+    }
+
+    return res;
+}
+
+
+/* optimized to use unchecked read_byte if there is definitely space */
+off_t frt_is_read_voff_t(FrtInStream *is)
+{
+    register frt_off_t res, b;
+    register int shift = 7;
+
+    if (is->buf.pos > (is->buf.len - VINT_MAX_LEN)) {
+        b = frt_is_read_byte(is);
+        res = b & 0x7F;                 /* 0x7F = 0b01111111 */
+
+        while ((b & 0x80) != 0) {       /* 0x80 = 0b10000000 */
+            b = frt_is_read_byte(is);
+            res |= (b & 0x7F) << shift;
+            shift += 7;
+        }
+    }
+    else {                              /* unchecked optimization */
+        b = read_byte(is);
+        res = b & 0x7F;                 /* 0x7F = 0b01111111 */
+
+        while ((b & 0x80) != 0) {       /* 0x80 = 0b10000000 */
+            b = read_byte(is);
+            res |= (b & 0x7F) << shift;
+            shift += 7;
+        }
+    }
+
+    return res;
+}
+
+/* optimized to use unchecked read_byte if there is definitely space */
+frt_u64 frt_is_read_vll(FrtInStream *is)
+{
+    register frt_u64 res, b;
+    register int shift = 7;
+
+    if (is->buf.pos > (is->buf.len - VINT_MAX_LEN)) {
+        b = frt_is_read_byte(is);
+        res = b & 0x7F;                 /* 0x7F = 0b01111111 */
+
+        while ((b & 0x80) != 0) {       /* 0x80 = 0b10000000 */
+            b = frt_is_read_byte(is);
+            res |= (b & 0x7F) << shift;
+            shift += 7;
+        }
+    }
+    else {                              /* unchecked optimization */
+        b = read_byte(is);
+        res = b & 0x7F;                 /* 0x7F = 0b01111111 */
+
+        while ((b & 0x80) != 0) {       /* 0x80 = 0b10000000 */
+            b = read_byte(is);
+            res |= (b & 0x7F) << shift;
+            shift += 7;
+        }
+    }
+
+    return res;
+}
+
+void frt_is_skip_vints(FrtInStream *is, register int cnt)
+{
+    for (; cnt > 0; cnt--) {
+        while ((frt_is_read_byte(is) & 0x80) != 0) {
+        }
+    }
+}
+
+char *frt_is_read_string(FrtInStream *is)
+{
+    register int length = (int) frt_is_read_vint(is);
+    char *str = FRT_ALLOC_N(char, length + 1);
+    str[length] = '\0';
+
+    if (is->buf.pos > (is->buf.len - length)) {
+        register int i;
+        for (i = 0; i < length; i++) {
+            str[i] = frt_is_read_byte(is);
+        }
+    }
+    else {                      /* unchecked optimization */
+        memcpy(str, is->buf.buf + is->buf.pos, length);
+        is->buf.pos += length;
+    }
+
+    return str;
+}
+
+char *frt_is_read_string_safe(FrtInStream *is)
+{
+    register int length = (int) frt_is_read_vint(is);
+    char *str = FRT_ALLOC_N(char, length + 1);
+    str[length] = '\0';
+
+    FRT_TRY
+        if (is->buf.pos > (is->buf.len - length)) {
+            register int i;
+            for (i = 0; i < length; i++) {
+                str[i] = frt_is_read_byte(is);
+            }
+        }
+        else {                      /* unchecked optimization */
+            memcpy(str, is->buf.buf + is->buf.pos, length);
+            is->buf.pos += length;
+        }
+    FRT_XCATCHALL
+        free(str);
+    FRT_XENDTRY
+
+    return str;
+}
+
+static char *frt_is_read_brotli_compressed_bytes(FrtInStream *is, int compressed_len, int *len) {
+    int buf_out_idx = 0;
+    int read_len;
+    frt_uchar buf_in[FRT_COMPRESSION_BUFFER_SIZE];
+    const frt_uchar *next_in;
+    size_t available_in;
+    frt_uchar *buf_out = NULL;
+    frt_uchar *next_out;
+    size_t available_out;
+
+    BrotliDecoderState *b_state = BrotliDecoderCreateInstance(NULL, NULL, NULL);
+    BrotliDecoderResult b_result = BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT;
+    if (!b_state) { frt_comp_raise(); return NULL; }
+
+    do {
+        read_len = (compressed_len > FRT_COMPRESSION_BUFFER_SIZE) ? FRT_COMPRESSION_BUFFER_SIZE : compressed_len;
+        frt_is_read_bytes(is, buf_in, read_len);
+        compressed_len -= read_len;
+        available_in = read_len;
+        next_in = buf_in;
+        available_out = FRT_COMPRESSION_BUFFER_SIZE;
+        do {
+            FRT_REALLOC_N(buf_out, frt_uchar, buf_out_idx + FRT_COMPRESSION_BUFFER_SIZE);
+            next_out = buf_out + buf_out_idx;
+            b_result = BrotliDecoderDecompressStream(b_state,
+                &available_in, &next_in,
+                &available_out, &next_out, NULL);
+            if (b_result == BROTLI_DECODER_RESULT_ERROR) { frt_comp_raise(); return NULL; }
+            buf_out_idx += FRT_COMPRESSION_BUFFER_SIZE - available_out;
+        } while (b_result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT);
+    } while (b_result != BROTLI_DECODER_RESULT_SUCCESS && compressed_len > 0);
+
+    BrotliDecoderDestroyInstance(b_state);
+
+    FRT_REALLOC_N(buf_out, frt_uchar, buf_out_idx + 1);
+    buf_out[buf_out_idx] = '\0';
+    *len = buf_out_idx;
+    return (char *)buf_out;
+}
+
+static char *frt_is_read_bz2_compressed_bytes(FrtInStream *is, int compressed_len, int *len) {
+    int buf_out_idx = 0, ret, read_len;
+    char *buf_out = NULL;
+    char buf_in[FRT_COMPRESSION_BUFFER_SIZE];
+    bz_stream zstrm;
+    zstrm.bzalloc = NULL;
+    zstrm.bzfree  = NULL;
+    zstrm.opaque  = NULL;
+    zstrm.next_in = NULL;
+    zstrm.avail_in = 0;
+    if ((ret = BZ2_bzDecompressInit(&zstrm, 0, 0)) != BZ_OK) frt_zraise(ret);
+
+    do {
+        read_len = (compressed_len > FRT_COMPRESSION_BUFFER_SIZE) ? FRT_COMPRESSION_BUFFER_SIZE : compressed_len;
+        frt_is_read_bytes(is, (frt_uchar *)buf_in, read_len);
+        compressed_len -= read_len;
+        zstrm.avail_in = read_len;
+        zstrm.next_in = buf_in;
+        zstrm.avail_out = FRT_COMPRESSION_BUFFER_SIZE;
+
+        do {
+            REALLOC_N(buf_out, char, buf_out_idx + FRT_COMPRESSION_BUFFER_SIZE);
+            zstrm.next_out = buf_out + buf_out_idx;
+            ret = BZ2_bzDecompress(&zstrm);
+            assert(ret != BZ_SEQUENCE_ERROR);  /* state not clobbered */
+            if (ret != BZ_OK && ret != BZ_STREAM_END) {
+                (void)BZ2_bzDecompressEnd(&zstrm);
+                frt_zraise(ret);
+            }
+            buf_out_idx += FRT_COMPRESSION_BUFFER_SIZE - zstrm.avail_out;
+        } while (zstrm.avail_out == 0);
+    } while (ret != BZ_STREAM_END && compressed_len != 0);
+
+    (void)BZ2_bzDecompressEnd(&zstrm);
+
+    FRT_REALLOC_N(buf_out, char, buf_out_idx + 1);
+    buf_out[buf_out_idx] = '\0';
+
+    *len = buf_out_idx;
+    return (char *)buf_out;
+}
+
+static char *frt_is_read_lz4_compressed_bytes(FrtInStream *is, int compressed_len, int *length) {
+    frt_uchar buf_in[FRT_COMPRESSION_BUFFER_SIZE];
+    char *buf_out = NULL;
+    int dc_length = 0;
+    LZ4F_dctx *dctx;
+    LZ4F_frameInfo_t frame_info;
+    LZ4F_errorCode_t dctx_status = LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION);
+    if (LZ4F_isError(dctx_status)) { *length = -1; return NULL; }
+
+    /* header and buffer */
+    int read_length = (compressed_len > FRT_COMPRESSION_BUFFER_SIZE) ? FRT_COMPRESSION_BUFFER_SIZE : compressed_len;
+    frt_is_read_bytes(is, buf_in, read_length);
+    compressed_len -= read_length;
+
+    size_t consumed_size = read_length;
+    size_t res = LZ4F_getFrameInfo(dctx, &frame_info, buf_in, &consumed_size);
+    if (LZ4F_isError(res)) { *length = -1; return NULL; }
+    size_t buf_out_length;
+    switch(frame_info.blockSizeID) {
+        case LZ4F_default:
+        case LZ4F_max64KB:
+            buf_out_length = 1 << 16;
+            break;
+        case LZ4F_max256KB:
+            buf_out_length = 1 << 18;
+            break;
+        case LZ4F_max1MB:
+            buf_out_length = 1 << 20;
+            break;
+        case LZ4F_max4MB:
+            buf_out_length = 1 << 22;
+            break;
+        default:
+            buf_out_length = 0;
+    }
+
+    res = 1;
+    int first_chunk = 1;
+
+    /* decompress data */
+    while (res != 0) {
+        if (!first_chunk) {
+            read_length = (compressed_len > FRT_COMPRESSION_BUFFER_SIZE) ? FRT_COMPRESSION_BUFFER_SIZE : compressed_len;
+            frt_is_read_bytes(is, buf_in, read_length);
+            compressed_len -= read_length;
+            consumed_size = 0;
+        }
+        first_chunk = 0;
+
+        char *src = (char *)(buf_in + consumed_size);
+        char *src_end = (char *)buf_in + read_length;
+
+        while (src < src_end && res != 0){
+            size_t dest_length = buf_out_length;
+            size_t consumed_size = read_length;
+            FRT_REALLOC_N(buf_out, char, dc_length + buf_out_length);
+            res = LZ4F_decompress(dctx, buf_out + dc_length, &dest_length, src, &consumed_size, NULL);
+            if (LZ4F_isError(res)) { *length = -1; return NULL; }
+            dc_length += dest_length;
+            src = src + consumed_size;
+        }
+    }
+
+    /* finish up */
+    LZ4F_freeDecompressionContext(dctx);
+
+    FRT_REALLOC_N(buf_out, char, dc_length + 1);
+    buf_out[dc_length] = '\0';
+
+    *length = dc_length;
+    return buf_out;
+}
+
+char *frt_is_read_compressed_bytes(FrtInStream *is, int compressed_len, int *len, FrtCompressionType compression) {
+    switch (compression) {
+        case FRT_COMPRESSION_BROTLI:
+            return frt_is_read_brotli_compressed_bytes(is, compressed_len, len);
+        case FRT_COMPRESSION_BZ2:
+            return frt_is_read_bz2_compressed_bytes(is, compressed_len, len);
+        case FRT_COMPRESSION_LZ4:
+            return frt_is_read_lz4_compressed_bytes(is, compressed_len, len);
+        default:
+            return NULL;
+    }
+}