isomorfeus-ferret 0.17.1 → 0.17.3

data/ext/isomorfeus_ferret_ext/frt_in_stream.c

@@ -0,0 +1,482 @@
+/* prevent warning: #warning Please include winsock2.h before windows.h [-Wcpp] */
+#ifdef _WIN32
+#   include <winsock2.h>
+#endif
+
+#include "brotli_decode.h"
+#include "brotli_encode.h"
+#include "bzlib.h"
+#include "lz4frame.h"
+#include "frt_except.h"
+#include "frt_in_stream.h"
+
+/**
+ * Create a newly allocated and initialized InStream
+ *
+ * @return a newly allocated and initialized InStream
+ */
+FrtInStream *frt_is_new(void) {
+    FrtInStream *is = FRT_ALLOC(FrtInStream);
+    is->folder_name = NULL;
+    is->f = FRT_ALLOC_AND_ZERO(FrtInStreamFile);
+    is->f->ref_cnt = 1;
+    is->buf.start = 0;
+    is->buf.pos = 0;
+    is->buf.len = 0;
+    is->ref_cnt = 1;
+    return is;
+}
+
+/**
+ * Refill the InStream's buffer from the store source (filesystem or memory).
+ *
+ * @param is the InStream to refill
+ * @raise FRT_IO_ERROR if there is a error reading from the filesystem
+ * @raise FRT_EOF_ERROR if there is an attempt to read past the end of the file
+ */
+static void frt_is_refill(FrtInStream *is) {
+    frt_off_t start = is->buf.start + is->buf.pos;
+    frt_off_t last = start + FRT_BUFFER_SIZE;
+    frt_off_t flen = is->m->length_i(is);
+
+    if (last > flen) {          /* don't read past EOF */
+        last = flen;
+    }
+
+    is->buf.len = last - start;
+    if (is->buf.len <= 0) {
+        FRT_RAISE(FRT_EOF_ERROR, "current pos = %"FRT_OFF_T_PFX"d, "
+              "file length = %"FRT_OFF_T_PFX"d", start, flen);
+    }
+
+    is->m->read_i(is, is->buf.buf, is->buf.len);
+
+    is->buf.start = start;
+    is->buf.pos = 0;
+}
+
+/**
+ * Unsafe alternative to frt_is_read_byte. Only use this method when you know
+ * there is no chance that you will read past the end of the InStream's
+ * buffer.
+ */
+#define read_byte(is) is->buf.buf[is->buf.pos++]
+
+/**
+ * Read a singly byte (unsigned char) from the InStream +is+.
+ *
+ * @param is the Instream to read from
+ * @return a single unsigned char read from the InStream +is+
+ * @raise FRT_IO_ERROR if there is a error reading from the filesystem
+ * @raise FRT_EOF_ERROR if there is an attempt to read past the end of the file
+ */
+frt_uchar frt_is_read_byte(FrtInStream *is) {
+    if (is->buf.pos >= is->buf.len) {
+        frt_is_refill(is);
+    }
+    return read_byte(is);
+}
+
+off_t frt_is_pos(FrtInStream *is) {
+    return is->buf.start + is->buf.pos;
+}
+
+frt_uchar *frt_is_read_bytes(FrtInStream *is, frt_uchar *buf, int len) {
+    int i;
+    frt_off_t start;
+
+    if ((is->buf.pos + len) < is->buf.len) {
+        for (i = 0; i < len; i++) {
+            buf[i] = read_byte(is);
+        }
+    } else {                              /* read all-at-once */
+        start = frt_is_pos(is);
+        is->m->seek_i(is, start);
+        is->m->read_i(is, buf, len);
+
+        is->buf.start = start + len;    /* adjust stream variables */
+        is->buf.pos = 0;
+        is->buf.len = 0;                /* trigger refill on read */
+    }
+    return buf;
+}
+
+void frt_is_seek(FrtInStream *is, frt_off_t pos) {
+    if (pos >= is->buf.start && pos < (is->buf.start + is->buf.len)) {
+        is->buf.pos = pos - is->buf.start;  /* seek within buffer */
+    } else {
+        is->buf.start = pos;
+        is->buf.pos = 0;
+        is->buf.len = 0;                    /* trigger refill() on read() */
+        is->m->seek_i(is, pos);
+    }
+}
+
+void frt_is_close(FrtInStream *is) {
+    if (is->ref_cnt == 0) {
+        FRT_RAISE(FRT_STATE_ERROR, "is ref_cnt to low\n");
+    }
+
+    if (FRT_DEREF(is) == 0) {
+        if (FRT_DEREF(is->f) == 0) {
+            is->m->close_i(is);
+            free(is->f);
+        }
+        if (is->folder_name) {
+            free((void *)is->folder_name);
+        }
+        free(is);
+    }
+}
+
+FrtInStream *frt_is_clone(FrtInStream *is) {
+    if (!(is->f))
+        return NULL;
+    FrtInStream *new_is = FRT_ALLOC(FrtInStream);
+    memcpy(new_is, is, sizeof(FrtInStream));
+    if (is->folder_name) {
+        new_is->folder_name = frt_estrdup(is->folder_name);
+    }
+    new_is->ref_cnt = 1;
+    FRT_REF(new_is->f);
+    return new_is;
+}
+
+frt_i32 frt_is_read_i32(FrtInStream *is) {
+    return ((frt_i32)frt_is_read_byte(is) << 24) |
+        ((frt_i32)frt_is_read_byte(is) << 16) |
+        ((frt_i32)frt_is_read_byte(is) << 8) |
+        ((frt_i32)frt_is_read_byte(is));
+}
+
+
+frt_i64 frt_is_read_i64(FrtInStream *is) {
+    return ((frt_i64)frt_is_read_byte(is) << 56) |
+        ((frt_i64)frt_is_read_byte(is) << 48) |
+        ((frt_i64)frt_is_read_byte(is) << 40) |
+        ((frt_i64)frt_is_read_byte(is) << 32) |
+        ((frt_i64)frt_is_read_byte(is) << 24) |
+        ((frt_i64)frt_is_read_byte(is) << 16) |
+        ((frt_i64)frt_is_read_byte(is) << 8) |
+        ((frt_i64)frt_is_read_byte(is));
+}
+
+
+frt_u32 frt_is_read_u32(FrtInStream *is) {
+    return ((frt_u32)frt_is_read_byte(is) << 24) |
+        ((frt_u32)frt_is_read_byte(is) << 16) |
+        ((frt_u32)frt_is_read_byte(is) << 8) |
+        ((frt_u32)frt_is_read_byte(is));
+}
+
+frt_u64 frt_is_read_u64(FrtInStream *is) {
+    return ((frt_u64)frt_is_read_byte(is) << 56) |
+        ((frt_u64)frt_is_read_byte(is) << 48) |
+        ((frt_u64)frt_is_read_byte(is) << 40) |
+        ((frt_u64)frt_is_read_byte(is) << 32) |
+        ((frt_u64)frt_is_read_byte(is) << 24) |
+        ((frt_u64)frt_is_read_byte(is) << 16) |
+        ((frt_u64)frt_is_read_byte(is) << 8) |
+        ((frt_u64)frt_is_read_byte(is));
+}
+
+
+/* optimized to use unchecked read_byte if there is definitely space */
+unsigned int frt_is_read_vint(FrtInStream *is) {
+    register unsigned int res, b;
+    register int shift = 7;
+
+    if (is->buf.pos > (is->buf.len - VINT_MAX_LEN)) {
+        b = frt_is_read_byte(is);
+        res = b & 0x7F;                 /* 0x7F = 0b01111111 */
+
+        while ((b & 0x80) != 0) {       /* 0x80 = 0b10000000 */
+            b = frt_is_read_byte(is);
+            res |= (b & 0x7F) << shift;
+            shift += 7;
+        }
+    } else {                              /* unchecked optimization */
+        b = read_byte(is);
+        res = b & 0x7F;                 /* 0x7F = 0b01111111 */
+
+        while ((b & 0x80) != 0) {       /* 0x80 = 0b10000000 */
+            b = read_byte(is);
+            res |= (b & 0x7F) << shift;
+            shift += 7;
+        }
+    }
+
+    return res;
+}
+
+
+/* optimized to use unchecked read_byte if there is definitely space */
+off_t frt_is_read_voff_t(FrtInStream *is) {
+    register frt_off_t res, b;
+    register int shift = 7;
+
+    if (is->buf.pos > (is->buf.len - VINT_MAX_LEN)) {
+        b = frt_is_read_byte(is);
+        res = b & 0x7F;                 /* 0x7F = 0b01111111 */
+
+        while ((b & 0x80) != 0) {       /* 0x80 = 0b10000000 */
+            b = frt_is_read_byte(is);
+            res |= (b & 0x7F) << shift;
+            shift += 7;
+        }
+    } else {                              /* unchecked optimization */
+        b = read_byte(is);
+        res = b & 0x7F;                 /* 0x7F = 0b01111111 */
+
+        while ((b & 0x80) != 0) {       /* 0x80 = 0b10000000 */
+            b = read_byte(is);
+            res |= (b & 0x7F) << shift;
+            shift += 7;
+        }
+    }
+
+    return res;
+}
+
+/* optimized to use unchecked read_byte if there is definitely space */
+frt_u64 frt_is_read_vll(FrtInStream *is) {
+    register frt_u64 res, b;
+    register int shift = 7;
+
+    if (is->buf.pos > (is->buf.len - VINT_MAX_LEN)) {
+        b = frt_is_read_byte(is);
+        res = b & 0x7F;                 /* 0x7F = 0b01111111 */
+
+        while ((b & 0x80) != 0) {       /* 0x80 = 0b10000000 */
+            b = frt_is_read_byte(is);
+            res |= (b & 0x7F) << shift;
+            shift += 7;
+        }
+    } else {                              /* unchecked optimization */
+        b = read_byte(is);
+        res = b & 0x7F;                 /* 0x7F = 0b01111111 */
+
+        while ((b & 0x80) != 0) {       /* 0x80 = 0b10000000 */
+            b = read_byte(is);
+            res |= (b & 0x7F) << shift;
+            shift += 7;
+        }
+    }
+
+    return res;
+}
+
+void frt_is_skip_vints(FrtInStream *is, register int cnt) {
+    for (; cnt > 0; cnt--) {
+        while ((frt_is_read_byte(is) & 0x80) != 0) {
+        }
+    }
+}
+
+char *frt_is_read_string(FrtInStream *is) {
+    register int length = (int) frt_is_read_vint(is);
+    char *str = FRT_ALLOC_N(char, length + 1);
+    str[length] = '\0';
+
+    if (is->buf.pos > (is->buf.len - length)) {
+        register int i;
+        for (i = 0; i < length; i++) {
+            str[i] = frt_is_read_byte(is);
+        }
+    } else {                      /* unchecked optimization */
+        memcpy(str, is->buf.buf + is->buf.pos, length);
+        is->buf.pos += length;
+    }
+
+    return str;
+}
+
+char *frt_is_read_string_safe(FrtInStream *is) {
+    register int length = (int) frt_is_read_vint(is);
+    char *str = FRT_ALLOC_N(char, length + 1);
+    str[length] = '\0';
+
+    FRT_TRY
+        if (is->buf.pos > (is->buf.len - length)) {
+            register int i;
+            for (i = 0; i < length; i++) {
+                str[i] = frt_is_read_byte(is);
+            }
+        } else {                      /* unchecked optimization */
+            memcpy(str, is->buf.buf + is->buf.pos, length);
+            is->buf.pos += length;
+        }
+    FRT_XCATCHALL
+        free(str);
+    FRT_XENDTRY
+
+    return str;
+}
+
+static char *frt_is_read_brotli_compressed_bytes(FrtInStream *is, int compressed_len, int *len) {
+    int buf_out_idx = 0;
+    int read_len;
+    frt_uchar buf_in[FRT_COMPRESSION_BUFFER_SIZE];
+    const frt_uchar *next_in;
+    size_t available_in;
+    frt_uchar *buf_out = NULL;
+    frt_uchar *next_out;
+    size_t available_out;
+
+    BrotliDecoderState *b_state = BrotliDecoderCreateInstance(NULL, NULL, NULL);
+    BrotliDecoderResult b_result = BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT;
+    if (!b_state) { frt_comp_raise(); return NULL; }
+
+    do {
+        read_len = (compressed_len > FRT_COMPRESSION_BUFFER_SIZE) ? FRT_COMPRESSION_BUFFER_SIZE : compressed_len;
+        frt_is_read_bytes(is, buf_in, read_len);
+        compressed_len -= read_len;
+        available_in = read_len;
+        next_in = buf_in;
+        available_out = FRT_COMPRESSION_BUFFER_SIZE;
+        do {
+            FRT_REALLOC_N(buf_out, frt_uchar, buf_out_idx + FRT_COMPRESSION_BUFFER_SIZE);
+            next_out = buf_out + buf_out_idx;
+            b_result = BrotliDecoderDecompressStream(b_state,
+                &available_in, &next_in,
+                &available_out, &next_out, NULL);
+            if (b_result == BROTLI_DECODER_RESULT_ERROR) { frt_comp_raise(); return NULL; }
+            buf_out_idx += FRT_COMPRESSION_BUFFER_SIZE - available_out;
+        } while (b_result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT);
+    } while (b_result != BROTLI_DECODER_RESULT_SUCCESS && compressed_len > 0);
+
+    BrotliDecoderDestroyInstance(b_state);
+
+    FRT_REALLOC_N(buf_out, frt_uchar, buf_out_idx + 1);
+    buf_out[buf_out_idx] = '\0';
+    *len = buf_out_idx;
+    return (char *)buf_out;
+}
+
+static char *frt_is_read_bz2_compressed_bytes(FrtInStream *is, int compressed_len, int *len) {
+    int buf_out_idx = 0, ret, read_len;
+    char *buf_out = NULL;
+    char buf_in[FRT_COMPRESSION_BUFFER_SIZE];
+    bz_stream zstrm;
+    zstrm.bzalloc = NULL;
+    zstrm.bzfree  = NULL;
+    zstrm.opaque  = NULL;
+    zstrm.next_in = NULL;
+    zstrm.avail_in = 0;
+    if ((ret = BZ2_bzDecompressInit(&zstrm, 0, 0)) != BZ_OK) frt_zraise(ret);
+
+    do {
+        read_len = (compressed_len > FRT_COMPRESSION_BUFFER_SIZE) ? FRT_COMPRESSION_BUFFER_SIZE : compressed_len;
+        frt_is_read_bytes(is, (frt_uchar *)buf_in, read_len);
+        compressed_len -= read_len;
+        zstrm.avail_in = read_len;
+        zstrm.next_in = buf_in;
+        zstrm.avail_out = FRT_COMPRESSION_BUFFER_SIZE;
+
+        do {
+            REALLOC_N(buf_out, char, buf_out_idx + FRT_COMPRESSION_BUFFER_SIZE);
+            zstrm.next_out = buf_out + buf_out_idx;
+            ret = BZ2_bzDecompress(&zstrm);
+            assert(ret != BZ_SEQUENCE_ERROR);  /* state not clobbered */
+            if (ret != BZ_OK && ret != BZ_STREAM_END) {
+                (void)BZ2_bzDecompressEnd(&zstrm);
+                frt_zraise(ret);
+            }
+            buf_out_idx += FRT_COMPRESSION_BUFFER_SIZE - zstrm.avail_out;
+        } while (zstrm.avail_out == 0);
+    } while (ret != BZ_STREAM_END && compressed_len != 0);
+
+    (void)BZ2_bzDecompressEnd(&zstrm);
+
+    FRT_REALLOC_N(buf_out, char, buf_out_idx + 1);
+    buf_out[buf_out_idx] = '\0';
+
+    *len = buf_out_idx;
+    return (char *)buf_out;
+}
+
+static char *frt_is_read_lz4_compressed_bytes(FrtInStream *is, int compressed_len, int *length) {
+    frt_uchar buf_in[FRT_COMPRESSION_BUFFER_SIZE];
+    char *buf_out = NULL;
+    int dc_length = 0;
+    LZ4F_dctx *dctx;
+    LZ4F_frameInfo_t frame_info;
+    LZ4F_errorCode_t dctx_status = LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION);
+    if (LZ4F_isError(dctx_status)) { *length = -1; return NULL; }
+
+    /* header and buffer */
+    int read_length = (compressed_len > FRT_COMPRESSION_BUFFER_SIZE) ? FRT_COMPRESSION_BUFFER_SIZE : compressed_len;
+    frt_is_read_bytes(is, buf_in, read_length);
+    compressed_len -= read_length;
+
+    size_t consumed_size = read_length;
+    size_t res = LZ4F_getFrameInfo(dctx, &frame_info, buf_in, &consumed_size);
+    if (LZ4F_isError(res)) { *length = -1; return NULL; }
+    size_t buf_out_length;
+    switch(frame_info.blockSizeID) {
+        case LZ4F_default:
+        case LZ4F_max64KB:
+            buf_out_length = 1 << 16;
+            break;
+        case LZ4F_max256KB:
+            buf_out_length = 1 << 18;
+            break;
+        case LZ4F_max1MB:
+            buf_out_length = 1 << 20;
+            break;
+        case LZ4F_max4MB:
+            buf_out_length = 1 << 22;
+            break;
+        default:
+            buf_out_length = 0;
+    }
+
+    res = 1;
+    int first_chunk = 1;
+
+    /* decompress data */
+    while (res != 0) {
+        if (!first_chunk) {
+            read_length = (compressed_len > FRT_COMPRESSION_BUFFER_SIZE) ? FRT_COMPRESSION_BUFFER_SIZE : compressed_len;
+            frt_is_read_bytes(is, buf_in, read_length);
+            compressed_len -= read_length;
+            consumed_size = 0;
+        }
+        first_chunk = 0;
+
+        char *src = (char *)(buf_in + consumed_size);
+        char *src_end = (char *)buf_in + read_length;
+
+        while (src < src_end && res != 0){
+            size_t dest_length = buf_out_length;
+            size_t consumed_size = read_length;
+            FRT_REALLOC_N(buf_out, char, dc_length + buf_out_length);
+            res = LZ4F_decompress(dctx, buf_out + dc_length, &dest_length, src, &consumed_size, NULL);
+            if (LZ4F_isError(res)) { *length = -1; return NULL; }
+            dc_length += dest_length;
+            src = src + consumed_size;
+        }
+    }
+
+    /* finish up */
+    LZ4F_freeDecompressionContext(dctx);
+
+    FRT_REALLOC_N(buf_out, char, dc_length + 1);
+    buf_out[dc_length] = '\0';
+
+    *length = dc_length;
+    return buf_out;
+}
+
+char *frt_is_read_compressed_bytes(FrtInStream *is, int compressed_len, int *len, FrtCompressionType compression_type) {
+    switch (compression_type) {
+        case FRT_COMPRESSION_BROTLI:
+            return frt_is_read_brotli_compressed_bytes(is, compressed_len, len);
+        case FRT_COMPRESSION_BZ2:
+            return frt_is_read_bz2_compressed_bytes(is, compressed_len, len);
+        case FRT_COMPRESSION_LZ4:
+            return frt_is_read_lz4_compressed_bytes(is, compressed_len, len);
+        default:
+            return NULL;
+    }
+}