brotli 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cd4d6db73c2704ff5b79264456b8ae9f371beb36
-  data.tar.gz: 33b074ac4545d43ed2aeb00713573754e24cb4c0
+  metadata.gz: f05525cdd6344037598d1f2ec2c1305e03d6bc72
+  data.tar.gz: 25c48ffa9319a43a73da655c11c5756caee19875
 SHA512:
-  metadata.gz: 536ba6e984a19db3e78f434f9f1b9b91cb90c5d7d1e0bbf84ee436382c05afee7e4912ae3ee03c37bbf732d9d59bc6be80f55dbf0fc245256c14cae264b87fad
-  data.tar.gz: 553ae97d40b5df932133af25c3075194a0754b56e0e2fe233302918b94c2cc0aeaf5373f18e74e3bb798bca68e3bbf30d34c35f04a21319880c6043e1052efe2
+  metadata.gz: e5e83a7e936a57db7949a2015517f3c8030b3051d8a7b4051f35e1c91d62e9889c982b6026817b9cf75c678f0881ea3fbeacfafe44fc8d841b69771a66da9874
+  data.tar.gz: 51f70a985a8684338a2c96465bac6147921c7531649b2d75517d739782fc28485735c6cd3e22cbf400bf2bf9b6346a8f54aede3291ae2d2bb4e02a4b974e9823

data/.gitignore

@@ -12,3 +12,6 @@
 *.o
 *.a
 mkmf.log
+/ext/brotli/common
+/ext/brotli/dec
+/ext/brotli/enc

data/.travis.yml

@@ -3,14 +3,22 @@ sudo: false
 cache:
   bundler: true
   directories:
-    - $HOME/.ccache
-
+    - "$HOME/.ccache"
 rvm:
   - 2.2.0
-  - 2.3.2
+  - 2.3.3
+  - 2.4.0
 env:
   global:
     - RANTLY_COUNT=10000
     - PATH="/usr/lib/ccache:$PATH"
 before_install:
   - gem install bundler
+deploy:
+  provider: rubygems
+  api_key:
+    secure: pHMMdI6cthdpQ8XHK4LPMRJK9yndRNRprSpttbK+M2Cq6O8ABIx9YsaiR4U6+kkS4RXwWnnJ6qkglQVLsCd1x+elJA0V8MxhOdzKyQgNC5wbIhtIdEZ9cN0c/hG8e9pKa35K9q9ETJ4M8QWNPxkJTRKw1yM8rIMbzf0oy0Btrcz+pJG8C9cMQwfV3QGJFPdO4Hcb5v2gK9rMDU5v3OD9CgbQqaZN3tbR2G1WTiAkFk86RlwPTyaurIRiUV/AQedPSPHHJ5SYbThjcu5iy1OtlFHkVTy/5QNm8ooJkX3pAVhnexTL/iufP0YI9xmYPq4k4RxK1ywSATbvz13JKyD94Sdlhpblz4kJaA4Fg3td8qQtZnFEICBuiW1yelWwVXbyb+zjeL8K4EizoqZV2WwKq2GR3fmyAgFHWiVXRbu0aNjJi23rpNkAGif41vAbsskgBgOiKYsW5/ZOE2opL7lH5l8GGjiYQ8cOKbiNKpUSFaCfwOwgZt4QBTNsN+GuasLNZkm3Dt4Wj7KhFJ+HtAfTnSpomsSQ5/wGwMUchQKo9mfSE0lmKglFyvRV23+WnEX0ZC2jUIJmaxGn1PajDL5tjnZOd5mZAehrc74r84o5olM39y7DynU6FP9QaVBaXF/vue4kIbL0ePSxxwMIYErOh0pOWbWx7ipMfDn+1jfsmvA=
+  gem: brotli
+  on:
+    tags: true
+    repo: miyucy/brotli

data/Gemfile

@@ -2,3 +2,5 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in brotli.gemspec
 gemspec
+
+gem 'travis'

data/ext/brotli/brotli.c

@@ -0,0 +1,279 @@
+#include "brotli.h"
+
+#define CSTR2SYM(x) ID2SYM(rb_intern(x))
+
+static VALUE rb_mBrotli;
+static VALUE rb_eBrotli;
+
+static void*
+brotli_alloc(void* opaque, size_t size)
+{
+    return malloc(size);
+}
+
+static void
+brotli_free(void* opaque, void* address)
+{
+    if (address) {
+        free(address);
+    }
+}
+
+/*******************************************************************************
+ * inflate
+ ******************************************************************************/
+
+typedef struct {
+    uint8_t* str;
+    size_t len;
+    BrotliDecoderState* s;
+    buffer_t* buffer;
+    BrotliDecoderResult r;
+} brotli_inflate_args_t;
+
+static void*
+brotli_inflate_no_gvl(void *arg)
+{
+    brotli_inflate_args_t *args = (brotli_inflate_args_t*)arg;
+    uint8_t         output[BUFSIZ];
+    BrotliDecoderResult  r = BROTLI_RESULT_ERROR;
+    size_t    available_in = args->len;
+    const uint8_t* next_in = args->str;
+    size_t   available_out = BUFSIZ;
+    uint8_t*      next_out = output;
+    size_t       total_out = 0;
+    buffer_t*       buffer = args->buffer;
+    BrotliDecoderState*  s = args->s;
+
+    for (;;) {
+        r = BrotliDecoderDecompressStream(s,
+                                          &available_in, &next_in,
+                                          &available_out, &next_out,
+                                          &total_out);
+        /* success, error or needs more input */
+        if (r != BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) {
+            break;
+        }
+        append_buffer(buffer, output, BUFSIZ);
+        available_out = BUFSIZ;
+        next_out = output;
+    }
+
+    if (r == BROTLI_DECODER_RESULT_SUCCESS) {
+        if (next_out != output) {
+            append_buffer(buffer, output, next_out - output);
+        }
+    }
+    args->r = r;
+
+    return arg;
+}
+
+static VALUE
+brotli_inflate(VALUE self, VALUE str)
+{
+    VALUE value = Qnil;
+    brotli_inflate_args_t args;
+
+    StringValue(str);
+
+    args.str = (uint8_t*)RSTRING_PTR(str);
+    args.len = (size_t)RSTRING_LEN(str);
+    args.buffer = create_buffer(BUFSIZ);
+    args.s = BrotliDecoderCreateInstance(brotli_alloc,
+                                         brotli_free,
+                                         NULL);
+    args.r = BROTLI_RESULT_ERROR;
+
+    rb_thread_call_without_gvl(brotli_inflate_no_gvl, (void *)&args, NULL, NULL);
+    if (args.r == BROTLI_DECODER_RESULT_SUCCESS) {
+        value = rb_str_new(args.buffer->ptr, args.buffer->used);
+        delete_buffer(args.buffer);
+        BrotliDecoderDestroyInstance(args.s);
+    } else if (args.r == BROTLI_DECODER_RESULT_ERROR) {
+        const char * error = BrotliDecoderErrorString(BrotliDecoderGetErrorCode(args.s));
+        delete_buffer(args.buffer);
+        BrotliDecoderDestroyInstance(args.s);
+        rb_raise(rb_eBrotli, "%s", error);
+    } else if (args.r == BROTLI_RESULT_NEEDS_MORE_INPUT) {
+        delete_buffer(args.buffer);
+        BrotliDecoderDestroyInstance(args.s);
+        rb_raise(rb_eBrotli, "Needs more input");
+    } else if (args.r == BROTLI_RESULT_NEEDS_MORE_OUTPUT) {
+        /* never reach to this block */
+        delete_buffer(args.buffer);
+        BrotliDecoderDestroyInstance(args.s);
+        rb_raise(rb_eBrotli, "Needs more output");
+    }
+
+    return value;
+}
+
+/*******************************************************************************
+ * deflate
+ ******************************************************************************/
+
+static void
+brotli_deflate_set_mode(BrotliEncoderState* s, VALUE value)
+{
+    if (NIL_P(value)) {
+        return;
+    } else {
+        if (value == CSTR2SYM("generic")) {
+            BrotliEncoderSetParameter(s, BROTLI_PARAM_MODE, BROTLI_MODE_GENERIC);
+        } else if (value == CSTR2SYM("text")) {
+            BrotliEncoderSetParameter(s, BROTLI_PARAM_MODE, BROTLI_MODE_TEXT);
+        } else if (value == CSTR2SYM("font")) {
+            BrotliEncoderSetParameter(s, BROTLI_PARAM_MODE, BROTLI_MODE_FONT);
+        } else {
+            rb_raise(rb_eArgError, "invalid mode");
+        }
+    }
+}
+
+static void
+brotli_deflate_set_quality(BrotliEncoderState* s, VALUE value)
+{
+    uint32_t param;
+    if (NIL_P(value)) {
+        return;
+    } else {
+        param = NUM2INT(value);
+        if (0 <= param && param <= 11) {
+            BrotliEncoderSetParameter(s, BROTLI_PARAM_QUALITY, param);
+        } else {
+            rb_raise(rb_eArgError, "invalid quality value. Should be 0 to 11.");
+        }
+    }
+}
+
+static void
+brotli_deflate_set_lgwin(BrotliEncoderState* s, VALUE value)
+{
+    uint32_t param;
+    if (NIL_P(value)) {
+        return;
+    } else {
+        param = NUM2INT(value);
+        if (10 <= param && param <= 24) {
+            BrotliEncoderSetParameter(s, BROTLI_PARAM_LGWIN, param);
+        } else {
+            rb_raise(rb_eArgError, "invalid lgwin value. Should be 10 to 24.");
+        }
+    }
+}
+
+static void
+brotli_deflate_set_lgblock(BrotliEncoderState* s, VALUE value)
+{
+    uint32_t param;
+    if (NIL_P(value)) {
+        return;
+    } else {
+        param = NUM2INT(value);
+        if ((param == 0) || (16 <= param && param <= 24)) {
+            BrotliEncoderSetParameter(s, BROTLI_PARAM_LGBLOCK, param);
+        } else {
+            rb_raise(rb_eArgError, "invalid lgblock value. Should be 0 or 16 to 24.");
+        }
+    }
+}
+
+static BrotliEncoderState*
+brotli_deflate_parse_options(BrotliEncoderState* s, VALUE opts)
+{
+    if (!NIL_P(opts)) {
+        brotli_deflate_set_mode(s, rb_hash_aref(opts, CSTR2SYM("mode")));
+        brotli_deflate_set_quality(s, rb_hash_aref(opts, CSTR2SYM("quality")));
+        brotli_deflate_set_lgwin(s, rb_hash_aref(opts, CSTR2SYM("lgwin")));
+        brotli_deflate_set_lgblock(s, rb_hash_aref(opts, CSTR2SYM("lgblock")));
+    }
+
+    return s;
+}
+
+typedef struct {
+    uint8_t *str;
+    size_t len;
+    BrotliEncoderState* s;
+    buffer_t* buffer;
+    BROTLI_BOOL finished;
+} brotli_deflate_args_t;
+
+static void*
+brotli_deflate_no_gvl(void *arg)
+{
+    brotli_deflate_args_t *args = (brotli_deflate_args_t *)arg;
+    uint8_t         output[BUFSIZ];
+    BROTLI_BOOL          r = BROTLI_FALSE;
+    size_t    available_in = args->len;
+    const uint8_t* next_in = args->str;
+    size_t   available_out = BUFSIZ;
+    uint8_t*      next_out = output;
+    size_t       total_out = 0;
+    buffer_t*       buffer = args->buffer;
+    BrotliEncoderState*  s = args->s;
+
+    for (;;) {
+        r = BrotliEncoderCompressStream(s,
+                                        BROTLI_OPERATION_FINISH,
+                                        &available_in, &next_in,
+                                        &available_out, &next_out, &total_out);
+        if (r == BROTLI_FALSE) {
+            args->finished = BROTLI_FALSE;
+            break;
+        } else {
+            append_buffer(buffer, output, next_out - output);
+            available_out = BUFSIZ;
+            next_out = output;
+
+            if (BrotliEncoderIsFinished(args->s)) {
+                args->finished = BROTLI_TRUE;
+                break;
+            }
+        }
+    }
+
+    return arg;
+}
+
+static VALUE
+brotli_deflate(int argc, VALUE *argv, VALUE self)
+{
+    VALUE str = Qnil, opts = Qnil, value = Qnil;
+    brotli_deflate_args_t args;
+
+    rb_scan_args(argc, argv, "11", &str, &opts);
+    StringValue(str);
+
+    args.str = (uint8_t*)RSTRING_PTR(str);
+    args.len = (size_t)RSTRING_LEN(str);
+    args.s = brotli_deflate_parse_options(
+        BrotliEncoderCreateInstance(brotli_alloc, brotli_free, NULL),
+        opts);
+    args.buffer = create_buffer(BUFSIZ);
+    args.finished = BROTLI_FALSE;
+
+    rb_thread_call_without_gvl(brotli_deflate_no_gvl, (void *)&args, NULL, NULL);
+    if (args.finished == BROTLI_TRUE) {
+        value = rb_str_new(args.buffer->ptr, args.buffer->used);
+    }
+
+    delete_buffer(args.buffer);
+    BrotliEncoderDestroyInstance(args.s);
+
+    return value;
+}
+
+/*******************************************************************************
+ * entry
+ ******************************************************************************/
+
+void
+Init_brotli(void)
+{
+    rb_mBrotli = rb_define_module("Brotli");
+    rb_eBrotli = rb_define_class_under(rb_mBrotli, "Error", rb_eStandardError);
+    rb_define_singleton_method(rb_mBrotli, "deflate", RUBY_METHOD_FUNC(brotli_deflate), -1);
+    rb_define_singleton_method(rb_mBrotli, "inflate", RUBY_METHOD_FUNC(brotli_inflate), 1);
+}

data/ext/brotli/brotli.h

@@ -2,7 +2,9 @@
 #define BROTLI_H 1
 
 #include "ruby.h"
+#include "ruby/thread.h"
 #include "enc/encode.h"
 #include "dec/decode.h"
+#include "buffer.h"
 
 #endif /* BROTLI_H */

data/ext/brotli/buffer.c

@@ -0,0 +1,95 @@
+#include "buffer.h"
+#define INITIAL (1024)
+
+buffer_t*
+create_buffer(const size_t initial) {
+    buffer_t *buffer = malloc(sizeof(buffer_t));
+    if (buffer == NULL) {
+        return NULL;
+    }
+
+    buffer->used = 0;
+    buffer->expand_count = 0;
+    buffer->expand_ratio = 130;
+    buffer->size = (size_t) (initial > 0 ? initial : INITIAL);
+    buffer->ptr = malloc(buffer->size);
+    if (buffer->ptr == NULL) {
+        delete_buffer(buffer);
+        return NULL;
+    }
+
+    return buffer;
+}
+
+void
+delete_buffer(buffer_t* buffer) {
+    if (buffer->ptr != NULL) {
+        free(buffer->ptr);
+        buffer->ptr = NULL;
+    }
+    free(buffer);
+}
+
+static
+buffer_t*
+expand_buffer(buffer_t* const buffer, const size_t need) {
+    size_t size = need * buffer->expand_ratio / 100;
+    uint8_t* ptr = malloc(size);
+    if (ptr == NULL) {
+        return NULL;
+    }
+    memcpy(ptr, buffer->ptr, buffer->size);
+    free(buffer->ptr);
+    buffer->ptr = ptr;
+    buffer->size = size;
+    buffer->expand_count += 1;
+    return buffer;
+}
+
+buffer_t*
+append_buffer(buffer_t* buffer, const void* ptr, const size_t size) {
+    if (buffer->used + size > buffer->size) {
+        if (expand_buffer(buffer, buffer->used + size) == NULL) {
+            return NULL;
+        }
+    }
+    memcpy(buffer->ptr + buffer->used, ptr, size);
+    buffer->used += size;
+    return buffer;
+}
+
+#if 0
+#include <stdio.h>
+
+void
+inspect_buffer(buffer_t* buffer) {
+    printf("ptr=%p size=%d used=%d expc=%d\n",
+           buffer->ptr,
+           buffer->size,
+           buffer->used,
+           buffer->expand_count);
+}
+
+int
+main(int argc, char *argv[])
+{
+    buffer_t* b = create_buffer(8);
+
+    inspect_buffer(b);
+    append_buffer(b, "1111", 4);
+    inspect_buffer(b);
+    append_buffer(b, "2222", 4);
+    inspect_buffer(b);
+    append_buffer(b, "3333", 4);
+    inspect_buffer(b);
+    append_buffer(b, "4444", 4);
+    inspect_buffer(b);
+    append_buffer(b, "5555", 4);
+    inspect_buffer(b);
+    append_buffer(b, "66", 2);
+    inspect_buffer(b);
+
+    delete_buffer(b);
+    return 0;
+}
+#endif

data/ext/brotli/buffer.h

@@ -0,0 +1,19 @@
+#ifndef __BUFFER_HEADER__
+#include <stdint.h>
+#include <stdlib.h>
+#include <stddef.h>
+#include <string.h>
+
+typedef struct {
+    uint8_t* ptr;
+    size_t size;
+    size_t used;
+    size_t expand_ratio;
+    size_t expand_count;
+} buffer_t;
+
+buffer_t* create_buffer(const size_t initial);
+void delete_buffer(buffer_t* buffer);
+buffer_t* append_buffer(buffer_t* buffer, const void* ptr, const size_t size);
+
+#endif // __BUFFER_HEADER__

data/ext/brotli/extconf.rb

@@ -3,94 +3,34 @@ require 'fileutils'
 require 'rbconfig'
 
 $CPPFLAGS << ' -DOS_MACOSX' if RbConfig::CONFIG['host_os'] =~ /darwin|mac os/
-$INCFLAGS << ' -I./enc -I./dec'
+$INCFLAGS << ' -I./enc -I./dec -I./common'
+create_makefile('brotli/brotli')
+
+def acopy(dir)
+  # source dir
+  FileUtils.mkdir_p File.expand_path(File.join(__dir__, dir))
+  # object dir
+  FileUtils.mkdir_p dir
+
+  files = Dir[File.expand_path(File.join(__dir__, '..', '..', 'vendor', 'brotli', dir, '*.[ch]'))]
+  FileUtils.cp files, dir, verbose: true
 
-FileUtils.mkdir_p 'enc'
-FileUtils.mkdir_p 'dec'
-FileUtils.mkdir_p File.join(__dir__, 'enc')
-FileUtils.mkdir_p File.join(__dir__, 'dec')
+  srcs = files.map { |e| File.basename e }.select { |e| e.end_with? '.c' }.map { |e| File.join(dir, e) }
+  objs = srcs.map { |e| e.sub(/\.c\z/, '.' + $OBJEXT) }
+  [srcs, objs]
+end
 
 srcs = []
 objs = []
-[
-  'dec/bit_reader.c',
-  'dec/bit_reader.h',
-  'dec/context.h',
-  'dec/decode.c',
-  'dec/decode.h',
-  'dec/dictionary.c',
-  'dec/dictionary.h',
-  'dec/huffman.c',
-  'dec/huffman.h',
-  'dec/port.h',
-  'dec/prefix.h',
-  'dec/state.c',
-  'dec/state.h',
-  'dec/transform.h',
-  'dec/types.h',
-  'enc/backward_references.cc',
-  'enc/backward_references.h',
-  'enc/bit_cost.h',
-  'enc/block_splitter.cc',
-  'enc/block_splitter.h',
-  'enc/brotli_bit_stream.cc',
-  'enc/brotli_bit_stream.h',
-  'enc/cluster.h',
-  'enc/command.h',
-  'enc/compress_fragment.cc',
-  'enc/compress_fragment.h',
-  'enc/compress_fragment_two_pass.cc',
-  'enc/compress_fragment_two_pass.h',
-  'enc/compressor.h',
-  'enc/context.h',
-  #'enc/dictionary.cc',
-  'enc/dictionary.h',
-  'enc/dictionary_hash.h',
-  'enc/encode.cc',
-  'enc/encode.h',
-  'enc/encode_parallel.cc',
-  'enc/encode_parallel.h',
-  'enc/entropy_encode.cc',
-  'enc/entropy_encode.h',
-  'enc/entropy_encode_static.h',
-  'enc/fast_log.h',
-  'enc/find_match_length.h',
-  'enc/hash.h',
-  'enc/histogram.cc',
-  'enc/histogram.h',
-  'enc/literal_cost.cc',
-  'enc/literal_cost.h',
-  'enc/metablock.cc',
-  'enc/metablock.h',
-  'enc/port.h',
-  'enc/prefix.h',
-  'enc/ringbuffer.h',
-  'enc/static_dict.cc',
-  'enc/static_dict.h',
-  'enc/static_dict_lut.h',
-  'enc/streams.cc',
-  'enc/streams.h',
-  'enc/transform.h',
-  'enc/types.h',
-  'enc/utf8_util.cc',
-  'enc/utf8_util.h',
-  'enc/write_bits.h',
-].each do |file|
-  FileUtils.cp(File.expand_path(File.join(__dir__, '..', '..', 'vendor', 'brotli', file)),
-               File.expand_path(File.join(__dir__, file)),
-               verbose: true)
-  if file.end_with?('.c') or file.end_with?('.cc')
-    srcs.push file
-    objs.push file.sub(/\.cc?\z/, '.' + $OBJEXT)
-  end
+%w(enc dec common).each do |dir|
+  a, b = acopy dir
+  srcs.concat a
+  objs.concat b
 end
 
-have_library('stdc++')
-create_makefile('brotli/brotli')
-
 File.open('Makefile', 'r+') do |f|
-  src = 'ORIG_SRCS = brotli.cc'
-  obj = 'OBJS = brotli.o'
+  src = 'ORIG_SRCS = brotli.c buffer.c'
+  obj = 'OBJS = brotli.o buffer.o'
   txt = f.read
         .sub(/^#{src}$/, src + ' ' + srcs.join(' '))
         .sub(/^#{obj}$/, obj + ' ' + objs.join(' '))

data/lib/brotli/version.rb

@@ -1,3 +1,3 @@
 module Brotli
-  VERSION = '0.1.3'
+  VERSION = '0.1.4'
 end

data/vendor/brotli/dec/bit_reader.c

@@ -8,8 +8,8 @@
 
 #include "./bit_reader.h"
 
+#include "../common/types.h"
 #include "./port.h"
-#include "./types.h"
 
 #if defined(__cplusplus) || defined(c_plusplus)
 extern "C" {
@@ -20,7 +20,7 @@ void BrotliInitBitReader(BrotliBitReader* const br) {
   br->bit_pos_ = sizeof(br->val_) << 3;
 }
 
-int BrotliWarmupBitReader(BrotliBitReader* const br) {
+BROTLI_BOOL BrotliWarmupBitReader(BrotliBitReader* const br) {
   size_t aligned_read_mask = (sizeof(br->val_) >> 1) - 1;
   /* Fixing alignment after unaligned BrotliFillWindow would result accumulator
      overflow. If unalignment is caused by BrotliSafeReadBits, then there is
@@ -30,17 +30,17 @@ int BrotliWarmupBitReader(BrotliBitReader* const br) {
   }
   if (BrotliGetAvailableBits(br) == 0) {
     if (!BrotliPullByte(br)) {
-      return 0;
+      return BROTLI_FALSE;
     }
   }
 
   while ((((size_t)br->next_in) & aligned_read_mask) != 0) {
     if (!BrotliPullByte(br)) {
       /* If we consumed all the input, we don't care about the alignment. */
-      return 1;
+      return BROTLI_TRUE;
     }
   }
-  return 1;
+  return BROTLI_TRUE;
 }
 
 #if defined(__cplusplus) || defined(c_plusplus)