isomorfeus-ferret 0.17.1 → 0.17.3

data/ext/isomorfeus_ferret_ext/frt_multimapper.c

@@ -6,13 +6,11 @@
 #define St(state) ((FrtState *)(state))
 #define UCtoI(val) ((int)(unsigned char)(val))
 
-static void state_destroy(FrtState *state)
-{
+static void state_destroy(FrtState *state) {
     state->destroy_i(state);
 }
 
-typedef struct LetterState
-{
+typedef struct LetterState {
     FrtState super;
     int c;
     int val;
@@ -21,30 +19,25 @@ typedef struct LetterState
 
 #define LSt(state) ((LetterState *)(state))
 
-static int lstate_next(LetterState *self, int c, int *states)
-{
+static int lstate_next(LetterState *self, int c, int *states) {
     if (c == self->c) {
         states[0] = self->val;
         return 1;
-    }
-    else {
+    } else {
         return 0;
     }
 }
 
-static int lstate_is_match(LetterState *self, char **mapping)
-{
+static int lstate_is_match(LetterState *self, char **mapping) {
     if (self->val < 0) {
         *mapping = self->mapping;
         return self->val;
-    }
-    else {
+    } else {
         return 0;
     }
 }
 
-static LetterState *lstate_new(int c, int val)
-{
+static LetterState *lstate_new(int c, int val) {
     LetterState *self   = FRT_ALLOC(LetterState);
     self->c             = c;
     self->val           = val;
@@ -55,28 +48,24 @@ static LetterState *lstate_new(int c, int val)
     return self;
 }
 
-typedef struct NonDeterministicState
-{
+typedef struct NonDeterministicState {
     FrtState super;
     int *states[256];
     int size[256];
     int capa[256];
 } NonDeterministicState;
 
-static int ndstate_next(NonDeterministicState *self, int c, int *states)
-{
+static int ndstate_next(NonDeterministicState *self, int c, int *states) {
     int size = self->size[c];
     memcpy(states, self->states[c], size * sizeof(int));
     return size;
 }
 
-static void ndstate_add(NonDeterministicState *self, int c, int state)
-{
+static void ndstate_add(NonDeterministicState *self, int c, int state) {
     if (self->capa[c] <= self->size[c]) {
         if (self->capa[c] == 0) {
             self->capa[c] = 4;
-        }
-        else {
+        } else {
             self->capa[c] <<= 1;
         }
         FRT_REALLOC_N(self->states[c], int, self->capa[c]);
@@ -92,8 +81,7 @@ static void ndstate_destroy_i(NonDeterministicState *self) {
     free(self);
 }
 
-static int ndstate_is_match(FrtState *self, char **mapping)
-{
+static int ndstate_is_match(FrtState *self, char **mapping) {
     (void)self; (void)mapping;
     return 0;
 }
@@ -118,8 +106,7 @@ FrtMultiMapper *frt_mulmap_new(void) {
     return self;
 }
 
-static void mulmap_free_dstates(FrtMultiMapper *self)
-{
+static void mulmap_free_dstates(FrtMultiMapper *self) {
     if (self->d_size > 0) {
         int i;
         for (i = self->d_size - 1; i >= 0; i--) {
@@ -129,8 +116,7 @@ static void mulmap_free_dstates(FrtMultiMapper *self)
     }
 }
 
-void frt_mulmap_add_mapping(FrtMultiMapper *self, const char *pattern, const char *rep)
-{
+void frt_mulmap_add_mapping(FrtMultiMapper *self, const char *pattern, const char *rep) {
     if (pattern == NULL || pattern[0] == '\0') {
         FRT_RAISE(FRT_ARG_ERROR, "Tried to add empty pattern to multi_mapper");
     } else {
@@ -147,16 +133,14 @@ void frt_mulmap_add_mapping(FrtMultiMapper *self, const char *pattern, const cha
 }
 
 
-static void mulmap_bv_set_states(FrtBitVector *bv, int *states, int cnt)
-{
+static void mulmap_bv_set_states(FrtBitVector *bv, int *states, int cnt) {
     int i;
     for (i = cnt - 1; i >= 0; i--) {
         frt_bv_set(bv, states[i]);
     }
 }
 
-static FrtDeterministicState *mulmap_process_state(FrtMultiMapper *self, FrtBitVector *bv)
-{
+static FrtDeterministicState *mulmap_process_state(FrtMultiMapper *self, FrtBitVector *bv) {
     FrtDeterministicState *current_state
         = (FrtDeterministicState *)frt_h_get(self->dstates_map, bv);
     if (current_state == NULL) {
@@ -197,15 +181,13 @@ static FrtDeterministicState *mulmap_process_state(FrtMultiMapper *self, FrtBitV
             }
             current_state->next[(int)c] = mulmap_process_state(self, nxt_bv);
         }
-    }
-    else {
+    } else {
         frt_bv_destroy(bv);
     }
     return current_state;
 }
 
-void frt_mulmap_compile(FrtMultiMapper *self)
-{
+void frt_mulmap_compile(FrtMultiMapper *self) {
     NonDeterministicState *start = ndstate_new();
     int i, j;
     int size = 1;
@@ -254,8 +236,7 @@ void frt_mulmap_compile(FrtMultiMapper *self)
     free(nstates);
 }
 
-int frt_mulmap_map_len(FrtMultiMapper *self, char *to, char *from, int capa)
-{
+int frt_mulmap_map_len(FrtMultiMapper *self, char *to, char *from, int capa) {
     FrtDeterministicState *start = self->dstates[0];
     FrtDeterministicState *state = start;
     char *s = from, *d = to, *end = to + capa - 1;
@@ -273,8 +254,7 @@ int frt_mulmap_map_len(FrtMultiMapper *self, char *to, char *from, int capa)
             memcpy(d, state->mapping, len);
             d += len;
             state = start;
-        }
-        else {
+        } else {
             *(d++) = *s;
         }
         s++;
@@ -283,15 +263,13 @@ int frt_mulmap_map_len(FrtMultiMapper *self, char *to, char *from, int capa)
     return d - to;
 }
 
-char *frt_mulmap_map(FrtMultiMapper *self, char *to, char *from, int capa)
-{
+char *frt_mulmap_map(FrtMultiMapper *self, char *to, char *from, int capa) {
     frt_mulmap_map_len(self, to, from, capa);
     return to;
 }
 
 /* Maps a string to a dynamically allocated string */
-char *frt_mulmap_dynamic_map(FrtMultiMapper *self, char *from)
-{
+char *frt_mulmap_dynamic_map(FrtMultiMapper *self, char *from) {
     FrtDeterministicState *start = self->dstates[0];
     FrtDeterministicState *state = start;
     int capa = strlen(from);
@@ -312,8 +290,7 @@ char *frt_mulmap_dynamic_map(FrtMultiMapper *self, char *from)
                 memcpy(d, state->mapping, len);
                 d += len;
                 state = start;
-            }
-            else {
+            } else {
                 *(d++) = *s;
             }
             s++;

data/ext/isomorfeus_ferret_ext/frt_multimapper.h

@@ -3,29 +3,25 @@
 
 #include "frt_hash.h"
 
-typedef struct FrtState
-{
+typedef struct FrtState {
     int  (*next)(struct FrtState *self, int c, int *states);
     void (*destroy_i)(struct FrtState *self);
     int  (*is_match)(struct FrtState *self, char **mapping);
 } FrtState;
 
-typedef struct FrtDeterministicState
-{
+typedef struct FrtDeterministicState {
     struct FrtDeterministicState *next[256];
     int longest_match;
     char *mapping;
     int mapping_len;
 } FrtDeterministicState;
 
-typedef struct FrtMapping
-{
+typedef struct FrtMapping {
     char *pattern;
     char *replacement;
 } FrtMapping;
 
-typedef struct FrtMultiMapper
-{
+typedef struct FrtMultiMapper {
     FrtMapping **mappings;
     int size;
     int capa;

data/ext/isomorfeus_ferret_ext/frt_out_stream.c

@@ -0,0 +1,334 @@
+/* prevent warning: #warning Please include winsock2.h before windows.h [-Wcpp] */
+#ifdef _WIN32
+#   include <winsock2.h>
+#endif
+
+#include "brotli_decode.h"
+#include "brotli_encode.h"
+#include "bzlib.h"
+#include "lz4frame.h"
+#include "frt_except.h"
+#include "frt_stream.h"
+#include "frt_out_stream.h"
+
+#define FRT_BROTLI_COMPRESSION_LEVEL 4
+#define FRT_BZIP_COMPRESSION_LEVEL 9
+
+/**
+ * Create a newly allocated and initialized OutStream object
+ *
+ * @return a newly allocated and initialized OutStream object
+ */
+FrtOutStream *frt_os_new(void) {
+    FrtOutStream *os = FRT_ALLOC(FrtOutStream);
+    os->folder_name = NULL;
+    os->buf.start = 0;
+    os->buf.pos = 0;
+    os->buf.len = 0;
+    return os;
+}
+
+/**
+ * Flush the countents of the FrtOutStream's buffers
+ *
+ * @param the OutStream to flush
+ */
+void frt_os_flush(FrtOutStream *os) {
+    os->m->flush_i(os, os->buf.buf, os->buf.pos);
+    os->buf.start += os->buf.pos;
+    os->buf.pos = 0;
+}
+
+void frt_os_close(FrtOutStream *os) {
+    frt_os_flush(os);
+    os->m->close_i(os);
+    if (os->folder_name) {
+        free((void *)os->folder_name);
+    }
+    free(os);
+}
+
+off_t frt_os_pos(FrtOutStream *os) {
+    return os->buf.start + os->buf.pos;
+}
+
+void frt_os_seek(FrtOutStream *os, frt_off_t new_pos) {
+    frt_os_flush(os);
+    os->buf.start = new_pos;
+    os->m->seek_i(os, new_pos);
+}
+
+/**
+ * Unsafe alternative to os_write_byte. Only use this method if you know there
+ * is no chance of buffer overflow.
+ */
+#define write_byte(os, b) os->buf.buf[os->buf.pos++] = (frt_uchar)b
+
+/**
+ * Write a single byte +b+ to the OutStream +os+
+ *
+ * @param os the OutStream to write to
+ * @param b  the byte to write
+ * @raise FRT_IO_ERROR if there is an IO error writing to the filesystem
+ */
+void frt_os_write_byte(FrtOutStream *os, frt_uchar b) {
+    if (os->buf.pos >= (FRT_BUFFER_SIZE - 1)) {
+        frt_os_flush(os);
+    }
+    write_byte(os, b);
+}
+
+void frt_os_write_bytes(FrtOutStream *os, const frt_uchar *buf, int len) {
+    if (len < (FRT_BUFFER_SIZE - os->buf.pos)) {
+        memcpy(os->buf.buf + os->buf.pos, buf, len);
+        os->buf.pos += len;
+    } else {
+        frt_os_flush(os);
+        int pos = 0;
+        int size;
+        while (pos < len) {
+            if (len - pos < FRT_BUFFER_SIZE) {
+                size = len - pos;
+            } else {
+                size = FRT_BUFFER_SIZE;
+            }
+            os->m->flush_i(os, buf + pos, size);
+            pos += size;
+            os->buf.start += size;
+        }
+    }
+}
+
+void frt_os_write_i32(FrtOutStream *os, frt_i32 num) {
+    frt_os_write_byte(os, (frt_uchar)((num >> 24) & 0xFF));
+    frt_os_write_byte(os, (frt_uchar)((num >> 16) & 0xFF));
+    frt_os_write_byte(os, (frt_uchar)((num >> 8) & 0xFF));
+    frt_os_write_byte(os, (frt_uchar)(num & 0xFF));
+}
+
+void frt_os_write_i64(FrtOutStream *os, frt_i64 num) {
+    frt_os_write_byte(os, (frt_uchar)((num >> 56) & 0xFF));
+    frt_os_write_byte(os, (frt_uchar)((num >> 48) & 0xFF));
+    frt_os_write_byte(os, (frt_uchar)((num >> 40) & 0xFF));
+    frt_os_write_byte(os, (frt_uchar)((num >> 32) & 0xFF));
+    frt_os_write_byte(os, (frt_uchar)((num >> 24) & 0xFF));
+    frt_os_write_byte(os, (frt_uchar)((num >> 16) & 0xFF));
+    frt_os_write_byte(os, (frt_uchar)((num >> 8) & 0xFF));
+    frt_os_write_byte(os, (frt_uchar)(num & 0xFF));
+}
+
+void frt_os_write_u32(FrtOutStream *os, frt_u32 num) {
+    frt_os_write_byte(os, (frt_uchar)((num >> 24) & 0xFF));
+    frt_os_write_byte(os, (frt_uchar)((num >> 16) & 0xFF));
+    frt_os_write_byte(os, (frt_uchar)((num >> 8) & 0xFF));
+    frt_os_write_byte(os, (frt_uchar)(num & 0xFF));
+}
+
+void frt_os_write_u64(FrtOutStream *os, frt_u64 num) {
+    frt_os_write_byte(os, (frt_uchar)((num >> 56) & 0xFF));
+    frt_os_write_byte(os, (frt_uchar)((num >> 48) & 0xFF));
+    frt_os_write_byte(os, (frt_uchar)((num >> 40) & 0xFF));
+    frt_os_write_byte(os, (frt_uchar)((num >> 32) & 0xFF));
+    frt_os_write_byte(os, (frt_uchar)((num >> 24) & 0xFF));
+    frt_os_write_byte(os, (frt_uchar)((num >> 16) & 0xFF));
+    frt_os_write_byte(os, (frt_uchar)((num >> 8) & 0xFF));
+    frt_os_write_byte(os, (frt_uchar)(num & 0xFF));
+}
+
+/* optimized to use an unchecked write if there is space */
+void frt_os_write_vint(FrtOutStream *os, register unsigned int num) {
+    if (os->buf.pos > VINT_END) {
+        while (num > 127) {
+            frt_os_write_byte(os, (frt_uchar)((num & 0x7f) | 0x80));
+            num >>= 7;
+        }
+        frt_os_write_byte(os, (frt_uchar)(num));
+    } else {
+        while (num > 127) {
+            write_byte(os, (frt_uchar)((num & 0x7f) | 0x80));
+            num >>= 7;
+        }
+        write_byte(os, (frt_uchar)(num));
+    }
+}
+
+/* optimized to use an unchecked write if there is space */
+void frt_os_write_voff_t(FrtOutStream *os, register frt_off_t num) {
+    if (os->buf.pos > VINT_END) {
+        while (num > 127) {
+            frt_os_write_byte(os, (frt_uchar)((num & 0x7f) | 0x80));
+            num >>= 7;
+        }
+        frt_os_write_byte(os, (frt_uchar)num);
+    } else {
+        while (num > 127) {
+            write_byte(os, (frt_uchar)((num & 0x7f) | 0x80));
+            num >>= 7;
+        }
+        write_byte(os, (frt_uchar)num);
+    }
+}
+
+/* optimized to use an unchecked write if there is space */
+void frt_os_write_vll(FrtOutStream *os, register frt_u64 num) {
+    if (os->buf.pos > VINT_END) {
+        while (num > 127) {
+            frt_os_write_byte(os, (frt_uchar)((num & 0x7f) | 0x80));
+            num >>= 7;
+        }
+        frt_os_write_byte(os, (frt_uchar)num);
+    } else {
+        while (num > 127) {
+            write_byte(os, (frt_uchar)((num & 0x7f) | 0x80));
+            num >>= 7;
+        }
+        write_byte(os, (frt_uchar)num);
+    }
+}
+
+void frt_os_write_string_len(FrtOutStream *os, const char *str, int len) {
+    frt_os_write_vint(os, len);
+    frt_os_write_bytes(os, (frt_uchar *)str, len);
+}
+
+void frt_os_write_string(FrtOutStream *os, const char *str) {
+    frt_os_write_string_len(os, str, (int)strlen(str));
+}
+
+static int frt_os_write_brotli_compressed_bytes(FrtOutStream* out_stream, frt_uchar *data, int length) {
+    size_t compressed_length = 0;
+    const frt_uchar *next_in = data;
+    size_t available_in = length;
+    size_t available_out;
+    frt_uchar compression_buffer[FRT_COMPRESSION_BUFFER_SIZE];
+    frt_uchar *next_out;
+    BrotliEncoderState *b_state = BrotliEncoderCreateInstance(NULL, NULL, NULL);
+    if (!b_state) { frt_comp_raise(); return -1; }
+
+    BrotliEncoderSetParameter(b_state, BROTLI_PARAM_QUALITY, FRT_BROTLI_COMPRESSION_LEVEL);
+
+    do {
+        available_out = FRT_COMPRESSION_BUFFER_SIZE;
+        next_out = compression_buffer;
+        if (!BrotliEncoderCompressStream(b_state, BROTLI_OPERATION_FINISH,
+            &available_in, &next_in,
+            &available_out, &next_out, &compressed_length)) {
+            BrotliEncoderDestroyInstance(b_state);
+            frt_comp_raise();
+            return -1;
+        }
+        frt_os_write_bytes(out_stream, compression_buffer, FRT_COMPRESSION_BUFFER_SIZE - available_out);
+    } while (!BrotliEncoderIsFinished(b_state));
+
+    BrotliEncoderDestroyInstance(b_state);
+
+    return (int)compressed_length;
+}
+
+static int frt_os_write_bz2_compressed_bytes(FrtOutStream* out_stream, frt_uchar *data, int length) {
+    int ret, buf_size, compressed_len = 0;
+    char out_buffer[FRT_COMPRESSION_BUFFER_SIZE];
+    bz_stream zstrm;
+    zstrm.bzalloc = NULL;
+    zstrm.bzfree  = NULL;
+    zstrm.opaque = NULL;
+    if ((ret = BZ2_bzCompressInit(&zstrm, FRT_BZIP_COMPRESSION_LEVEL, 0, 0)) != BZ_OK) frt_zraise(ret);
+
+    zstrm.avail_in = length;
+    zstrm.next_in = (char *)data;
+    zstrm.avail_out = FRT_COMPRESSION_BUFFER_SIZE;
+    zstrm.next_out = out_buffer;
+
+    do {
+        ret = BZ2_bzCompress(&zstrm, BZ_FINISH); /* no bad return value */
+        assert(ret != BZ_SEQUENCE_ERROR);        /* state not clobbered */
+        compressed_len += buf_size = FRT_COMPRESSION_BUFFER_SIZE - zstrm.avail_out;
+        frt_os_write_bytes(out_stream, (frt_uchar *)out_buffer, buf_size);
+    } while (zstrm.avail_out == 0);
+    assert(zstrm.avail_in == 0);       /* all input will be used */
+
+    (void)BZ2_bzCompressEnd(&zstrm);
+    return compressed_len;
+}
+
+static const LZ4F_preferences_t lz4_prefs = {
+    {
+        LZ4F_default,
+        LZ4F_blockLinked,
+        LZ4F_noContentChecksum,
+        LZ4F_frame,
+        0, /* unknown content size */
+        0, /* no dictID */
+        LZ4F_noBlockChecksum
+    },
+    0,
+    1,
+    1,
+    {0,0,0}
+};
+
+static int frt_os_write_lz4_compressed_bytes(FrtOutStream* out_stream, frt_uchar *data, int length) {
+    int compressed_length = 0;
+    int remaining_length = length;
+    size_t ccmp_length = 0;
+    LZ4F_compressionContext_t ctx;
+    size_t out_buf_length = LZ4F_compressBound(FRT_COMPRESSION_BUFFER_SIZE, &lz4_prefs);
+    frt_uchar *out_buf = frt_ecalloc(out_buf_length);
+
+    size_t ctx_creation = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION);
+    if (LZ4F_isError(ctx_creation)) {
+        compressed_length = -1;
+        goto finish;
+    }
+
+    /* create header */
+    ccmp_length = LZ4F_compressBegin(ctx, out_buf, out_buf_length, &lz4_prefs);
+    if (LZ4F_isError(ccmp_length)) {
+        compressed_length = -1;
+        goto finish;
+    }
+    compressed_length = ccmp_length;
+    frt_os_write_bytes(out_stream, out_buf, ccmp_length);
+
+    /* compress data */
+    do {
+        int read_length = (FRT_COMPRESSION_BUFFER_SIZE > remaining_length) ? remaining_length : FRT_COMPRESSION_BUFFER_SIZE;
+        ccmp_length = LZ4F_compressUpdate(ctx, out_buf, out_buf_length, data + (length - remaining_length), read_length, NULL);
+        if (LZ4F_isError(ccmp_length)) {
+            compressed_length = -1;
+            goto finish;
+        }
+        frt_os_write_bytes(out_stream, out_buf, ccmp_length);
+        compressed_length += ccmp_length;
+        remaining_length -= read_length;
+    } while (remaining_length > 0);
+
+    /* finish up */
+    ccmp_length = LZ4F_compressEnd(ctx, out_buf, out_buf_length, NULL);
+    if (LZ4F_isError(ccmp_length)) {
+        compressed_length = -1;
+        goto finish;
+    }
+
+    frt_os_write_bytes(out_stream, out_buf, ccmp_length);
+    compressed_length += ccmp_length;
+
+finish:
+    LZ4F_freeCompressionContext(ctx);
+    free(out_buf);
+
+    return compressed_length;
+}
+
+int frt_os_write_compressed_bytes(FrtOutStream* out_stream, frt_uchar *data, int length, FrtCompressionType compression_type) {
+    switch (compression_type) {
+        case FRT_COMPRESSION_BROTLI:
+            return frt_os_write_brotli_compressed_bytes(out_stream, data, length);
+        case FRT_COMPRESSION_BZ2:
+            return frt_os_write_bz2_compressed_bytes(out_stream, data, length);
+        case FRT_COMPRESSION_LZ4:
+            return frt_os_write_lz4_compressed_bytes(out_stream, data, length);
+        default:
+            return -1;
+    }
+}