brotli 0.7.0 → 0.9.0

data/ext/brotli/brotli.h

@@ -3,16 +3,11 @@
 
 #include "ruby.h"
 
-// ruby/thread.h is ruby 2.x
-#ifdef HAVE_RUBY_THREAD_H
 #include "ruby/thread.h"
-#endif
 
 #include "brotli/encode.h"
 #include "brotli/decode.h"
-#ifdef HAVE_BROTLI_SHARED_DICTIONARY_H
 #include "brotli/shared_dictionary.h"
-#endif
 #include "buffer.h"
 
 #endif /* BROTLI_H */

data/ext/brotli/buffer.c

@@ -1,90 +1,68 @@
 #include "buffer.h"
 #include "ruby/ruby.h"
-#define INITIAL (1024)
 
-buffer_t*
-create_buffer(const size_t initial) {
-    buffer_t *buffer = ruby_xmalloc(sizeof(buffer_t));
-    if (buffer == NULL) {
-        return NULL;
-    }
+#include <stdint.h>
+#include <string.h>
 
+#define BUFFER_INITIAL_SIZE 1024
+
+buffer_t*
+create_buffer(size_t initial)
+{
+    buffer_t *buffer = ruby_xmalloc(sizeof(*buffer));
     buffer->used = 0;
-    buffer->expand_count = 0;
-    buffer->expand_ratio = 130;
-    buffer->size = (size_t) (initial > 0 ? initial : INITIAL);
+    buffer->size = initial > 0 ? initial : BUFFER_INITIAL_SIZE;
     buffer->ptr = ruby_xmalloc(buffer->size);
-    if (buffer->ptr == NULL) {
-        delete_buffer(buffer);
-        return NULL;
-    }
-
     return buffer;
 }
 
 void
-delete_buffer(buffer_t* buffer) {
-    if (buffer->ptr != NULL) {
-        ruby_xfree(buffer->ptr);
-        buffer->ptr = NULL;
+delete_buffer(buffer_t* buffer)
+{
+    if (!buffer) {
+        return;
     }
+
+    ruby_xfree(buffer->ptr);
     ruby_xfree(buffer);
 }
 
-static
-buffer_t*
-expand_buffer(buffer_t* const buffer, const size_t need) {
-    size_t size = need * buffer->expand_ratio / 100;
-    buffer->ptr = ruby_xrealloc(buffer->ptr, size);
-    buffer->size = size;
-    buffer->expand_count += 1;
-    return buffer;
-}
+static size_t
+buffer_size_for(size_t current, size_t required)
+{
+    size_t size = current > 0 ? current : BUFFER_INITIAL_SIZE;
 
-buffer_t*
-append_buffer(buffer_t* buffer, const void* ptr, const size_t size) {
-    if (buffer->used + size > buffer->size) {
-        if (expand_buffer(buffer, buffer->used + size) == NULL) {
-            return NULL;
+    while (size < required) {
+        if (size > SIZE_MAX / 2) {
+            return required;
         }
+        size *= 2;
     }
-    memcpy(buffer->ptr + buffer->used, ptr, size);
-    buffer->used += size;
-    return buffer;
-}
 
-#if 0
-#include <stdio.h>
+    return size;
+}
 
-void
-inspect_buffer(buffer_t* buffer) {
-    printf("ptr=%p size=%d used=%d expc=%d\n",
-           buffer->ptr,
-           buffer->size,
-           buffer->used,
-           buffer->expand_count);
+static void
+expand_buffer(buffer_t* buffer, size_t required)
+{
+    buffer->size = buffer_size_for(buffer->size, required);
+    buffer->ptr = ruby_xrealloc(buffer->ptr, buffer->size);
 }
 
-int
-main(int argc, char *argv[])
+void
+append_buffer(buffer_t* buffer, const void* ptr, size_t size)
 {
-    buffer_t* b = create_buffer(8);
+    size_t required;
+
+    if (size == 0) {
+        return;
+    }
 
-    inspect_buffer(b);
-    append_buffer(b, "1111", 4);
-    inspect_buffer(b);
-    append_buffer(b, "2222", 4);
-    inspect_buffer(b);
-    append_buffer(b, "3333", 4);
-    inspect_buffer(b);
-    append_buffer(b, "4444", 4);
-    inspect_buffer(b);
-    append_buffer(b, "5555", 4);
-    inspect_buffer(b);
-    append_buffer(b, "66", 2);
-    inspect_buffer(b);
+    required = buffer->used + size;
+    if (required > buffer->size) {
+        expand_buffer(buffer, required);
+    }
 
-    delete_buffer(b);
-    return 0;
+    memcpy(buffer->ptr + buffer->used, ptr, size);
+    buffer->used += size;
 }
-#endif

data/ext/brotli/buffer.h

@@ -1,19 +1,16 @@
-#ifndef __BUFFER_HEADER__
-#include <stdint.h>
-#include <stdlib.h>
+#ifndef BUFFER_H
+#define BUFFER_H 1
+
 #include <stddef.h>
-#include <string.h>
 
 typedef struct {
     char* ptr;
     size_t size;
     size_t used;
-    size_t expand_ratio;
-    size_t expand_count;
 } buffer_t;
 
-buffer_t* create_buffer(const size_t initial);
+buffer_t* create_buffer(size_t initial);
 void delete_buffer(buffer_t* buffer);
-buffer_t* append_buffer(buffer_t* buffer, const void* ptr, const size_t size);
+void append_buffer(buffer_t* buffer, const void* ptr, size_t size);
 
-#endif // __BUFFER_HEADER__
+#endif /* BUFFER_H */

data/ext/brotli/extconf.rb

@@ -1,66 +1,131 @@
+# frozen_string_literal: true
+
 require "mkmf"
 require "fileutils"
 require "rbconfig"
+require "rubygems/version"
 
-dir_config("brotli")
+VENDORED_DIRS = %w[common dec enc include].freeze
+MINIMUM_BROTLI_VERSION = Gem::Version.new("1.2.0")
+BUILD_DIR = Dir.pwd
+VENDORED_SOURCE_DIR = File.expand_path("../../vendor/brotli/c", __dir__)
 
-# libbrotli-dev
-have_dev_pkg = [
-  have_header("brotli/decode.h"),
-  have_header("brotli/encode.h"),
-  pkg_config("libbrotlicommon"),
-  pkg_config("libbrotlidec"),
-  pkg_config("libbrotlienc")
-].all? { |e| e }
-
-have_header("brotli/shared_dictionary.h")
-have_func("BrotliEncoderPrepareDictionary", "brotli/encode.h")
-have_func("BrotliEncoderAttachPreparedDictionary", "brotli/encode.h")
-have_func("BrotliDecoderAttachDictionary", "brotli/decode.h")
-
-if enable_config("vendor")
-  have_dev_pkg = false
-  Logging::message "Use vendor brotli\n"
-  $defs << "-DHAVE_BROTLI_SHARED_DICTIONARY_H"
-  $defs << "-DHAVE_BROTLIENCODERPREPAREDICTIONARY"
-  $defs << "-DHAVE_BROTLIENCODERATTACHPREPAREDDICTIONARY"
-  $defs << "-DHAVE_BROTLIDECODERATTACHDICTIONARY"
+def generated_vendored_dirs
+  VENDORED_DIRS.map { |dirname| File.join(BUILD_DIR, dirname) }
 end
 
-$CPPFLAGS << " -DOS_MACOSX" if RbConfig::CONFIG["host_os"] =~ /darwin|mac os/
-$INCFLAGS << " -I$(srcdir)/enc -I$(srcdir)/dec -I$(srcdir)/common -I$(srcdir)/include" unless have_dev_pkg
+def remove_generated_vendored_dirs
+  generated_vendored_dirs.each { |path| FileUtils.rm_rf(path) }
+end
 
-create_makefile("brotli/brotli")
+def copy_vendored_sources
+  sources = Dir[File.join(VENDORED_SOURCE_DIR, "{common,dec,enc,include}", "**", "*.{c,h}")].sort
 
-unless have_dev_pkg
-  __DIR__ = File.expand_path(File.dirname(__FILE__))
+  if sources.empty?
+    abort "ERROR: Vendored Brotli sources not found at #{VENDORED_SOURCE_DIR}.\n" \
+          "Install system libbrotli-dev >= #{MINIMUM_BROTLI_VERSION} or ensure vendor sources are present."
+  end
 
-  %w[enc dec common include].each do |dirname|
-    FileUtils.rm_rf dirname
-    FileUtils.mkdir_p dirname
-    FileUtils.cp_r(
-      File.expand_path(File.join(__DIR__, "..", "..", "vendor", "brotli", "c", dirname), __DIR__),
-      __DIR__,
-      verbose: true
-    )
+  sources.each do |source|
+    relative_path = source.delete_prefix("#{VENDORED_SOURCE_DIR}/")
+    destination = File.join(BUILD_DIR, relative_path)
+
+    FileUtils.mkdir_p(File.dirname(destination))
+    FileUtils.cp(source, destination)
   end
+end
 
-  srcs = []
-  objs = []
-  Dir[File.expand_path(File.join("{enc,dec,common,include}", "**", "*.c"), __DIR__)].sort.each do |file|
-    file[__DIR__ + File::SEPARATOR] = ""
-    srcs << file
-    objs << file.sub(/\.c\z/, "." + RbConfig::CONFIG["OBJEXT"])
+def vendored_source_files
+  Dir[File.join(BUILD_DIR, "{common,dec,enc,include}", "**", "*.c")].sort.map do |path|
+    path.delete_prefix("#{BUILD_DIR}/")
   end
+end
+
+def add_vendored_sources_to_makefile
+  objext = RbConfig::CONFIG["OBJEXT"]
+  source_files = vendored_source_files
+  object_files = source_files.map { |path| path.sub(/\.c\z/, ".#{objext}") }
+
+  makefile = File.read("Makefile")
+  makefile = makefile
+             .sub(/^ORIG_SRCS = .*$/, "ORIG_SRCS = brotli.c buffer.c #{source_files.join(" ")}")
+             .sub(/^OBJS = .*$/, "OBJS = brotli.#{objext} buffer.#{objext} #{object_files.join(" ")}")
+  File.write("Makefile", makefile)
+end
+
+def mkmf_flags
+  {
+    cflags: $CFLAGS.dup,
+    cxxflags: $CXXFLAGS.dup,
+    cppflags: $CPPFLAGS.dup,
+    defs: $defs.dup,
+    incflags: $INCFLAGS.dup,
+    ldflags: $LDFLAGS.dup,
+    libs: $libs.dup,
+    local_libs: $LOCAL_LIBS.dup
+  }
+end
+
+def restore_mkmf_flags(flags)
+  $CFLAGS = flags[:cflags]
+  $CXXFLAGS = flags[:cxxflags]
+  $CPPFLAGS = flags[:cppflags]
+  $defs = flags[:defs]
+  $INCFLAGS = flags[:incflags]
+  $LDFLAGS = flags[:ldflags]
+  $libs = flags[:libs]
+  $LOCAL_LIBS = flags[:local_libs]
+end
 
-  File.open("Makefile", "r+") do |f|
-    obj_ext = RbConfig::CONFIG["OBJEXT"]
-    src = "ORIG_SRCS = brotli.c buffer.c"
-    obj = "OBJS = brotli.#{obj_ext} buffer.#{obj_ext}"
-    txt = f.read
-           .sub(/^ORIG_SRCS = .*$/, "#{src} #{srcs.join(" ")}")
-           .sub(/^OBJS = .*$/, "#{obj} #{objs.join(" ")}")
-    f.rewind
-    f.write txt
+def system_brotli_version
+  version = pkg_config("libbrotlicommon", "modversion")
+  Gem::Version.new(version) if version
+rescue ArgumentError
+  nil
+end
+
+def system_brotli_available?
+  flags = mkmf_flags
+  version = system_brotli_version
+
+  unless version && version >= MINIMUM_BROTLI_VERSION
+    Logging.message(
+      "System Brotli version %s is too old; need >= %s\n",
+      version || "unknown",
+      MINIMUM_BROTLI_VERSION
+    )
+    restore_mkmf_flags(flags)
+    return false
   end
+
+  available = pkg_config("libbrotlicommon") &&
+              pkg_config("libbrotlidec") &&
+              pkg_config("libbrotlienc") &&
+              have_header("brotli/decode.h") &&
+              have_header("brotli/encode.h") &&
+              have_header("brotli/shared_dictionary.h") &&
+              have_func("BrotliEncoderPrepareDictionary", "brotli/encode.h") &&
+              have_func("BrotliEncoderAttachPreparedDictionary", "brotli/encode.h") &&
+              have_func("BrotliDecoderAttachDictionary", "brotli/decode.h")
+
+  restore_mkmf_flags(flags) unless available
+  available
+end
+
+dir_config("brotli")
+have_func("rb_gc_mark_movable")
+
+$CPPFLAGS << " -DOS_MACOSX" if RbConfig::CONFIG["host_os"] =~ /darwin|mac os/
+
+remove_generated_vendored_dirs
+
+use_vendored_brotli = enable_config("vendor", false) || !system_brotli_available?
+
+if use_vendored_brotli
+  Logging.message "Using vendored Brotli sources\n"
+  copy_vendored_sources
+  $INCFLAGS << " -Ienc -Idec -Icommon -Iinclude"
 end
+
+create_makefile("brotli/brotli")
+add_vendored_sources_to_makefile if use_vendored_brotli

data/lib/brotli/reader.rb

@@ -0,0 +1,167 @@
+# frozen_string_literal: true
+
+module Brotli
+  class Reader
+    DEFAULT_READ_SIZE = 8192
+
+    def initialize(io, options = nil, **kwargs)
+      raise ArgumentError, "io should not be nil" if io.nil?
+
+      @io = io
+      @decompressor = Decompressor.new(normalize_options(options, kwargs))
+      @output_buffer = +""
+      @closed = false
+      @finished = false
+    end
+
+    def read(length = nil, outbuf = nil)
+      ensure_open
+
+      if length.nil?
+        drain_stream
+        return replace_outbuf(outbuf, take_output)
+      end
+
+      len = Integer(length)
+      raise ArgumentError, "negative length #{len} given" if len.negative?
+      return replace_outbuf(outbuf, +"") if len.zero?
+
+      fill_buffer(len, output_limit: len)
+      if @output_buffer.empty? && @finished
+        replace_outbuf(outbuf, +"") if outbuf
+        return nil
+      end
+
+      replace_outbuf(outbuf, take_output(len))
+    end
+
+    def readpartial(maxlen, outbuf = nil)
+      ensure_open
+
+      len = Integer(maxlen)
+      raise ArgumentError, "max length must be positive" unless len.positive?
+
+      until @finished || !@output_buffer.empty?
+        fill_buffer(1, output_limit: len, stop_after_output: true)
+      end
+
+      if @output_buffer.empty?
+        replace_outbuf(outbuf, +"") if outbuf
+        raise EOFError, "end of file reached"
+      end
+
+      replace_outbuf(outbuf, take_output(len))
+    end
+
+    def eof?
+      ensure_open
+
+      return false unless @output_buffer.empty?
+
+      fill_buffer(1, output_limit: 1, stop_after_output: true) unless @finished
+      @output_buffer.empty? && @finished
+    end
+
+    def close
+      return @io if @closed
+
+      begin
+        @io.close if @io.respond_to?(:close)
+      ensure
+        @closed = true
+        @finished = true
+        @decompressor = nil
+        @output_buffer.clear
+      end
+
+      @io
+    end
+
+    def closed?
+      @closed
+    end
+
+    private
+
+    def ensure_open
+      raise Error, "Reader is closed" if @closed
+    end
+
+    def replace_outbuf(outbuf, string)
+      return string unless outbuf
+
+      buffer = coerce_string(outbuf)
+      buffer.replace(string)
+      buffer
+    end
+
+    def take_output(length = @output_buffer.bytesize)
+      return +"".b if length <= 0 || @output_buffer.empty?
+
+      if length >= @output_buffer.bytesize
+        output = @output_buffer
+        @output_buffer = +""
+        return output
+      end
+
+      @output_buffer.slice!(0, length)
+    end
+
+    def drain_stream
+      fill_buffer(@output_buffer.bytesize + 1) until @finished
+    end
+
+    def fill_buffer(wanted, output_limit: nil, stop_after_output: false)
+      while @output_buffer.bytesize < wanted && !@finished
+        buffered = @output_buffer.bytesize
+        chunk = next_chunk
+        feed_chunk(chunk || +"", output_limit: remaining_output_limit(output_limit))
+        break if stop_after_output && !@output_buffer.empty?
+
+        next unless chunk.nil? || chunk.empty?
+        next if @finished || @output_buffer.bytesize > buffered || !@decompressor.can_accept_more_data
+
+        raise Error, "Unexpected end of compressed stream"
+      end
+    end
+
+    def feed_chunk(chunk, output_limit: nil)
+      output = if output_limit&.positive?
+        @decompressor.process(chunk, output_buffer_limit: output_limit)
+      else
+        @decompressor.process(chunk)
+      end
+
+      @output_buffer << output unless output.empty?
+      @finished = @decompressor.finished?
+    end
+
+    def next_chunk
+      return +"".b unless @decompressor.can_accept_more_data
+
+      @io.respond_to?(:readpartial) ? @io.readpartial(DEFAULT_READ_SIZE) : @io.read(DEFAULT_READ_SIZE)
+    rescue EOFError
+      nil
+    end
+
+    def remaining_output_limit(output_limit)
+      return unless output_limit
+
+      [output_limit - @output_buffer.bytesize, 0].max
+    end
+
+    def normalize_options(options, kwargs)
+      return kwargs if options.nil?
+
+      unless options.is_a?(Hash)
+        raise TypeError, "no implicit conversion of #{options.class} into Hash"
+      end
+
+      kwargs.empty? ? options : options.merge(kwargs)
+    end
+
+    def coerce_string(value)
+      String.try_convert(value) || raise(TypeError, "no implicit conversion of #{value.class} into String")
+    end
+  end
+end

data/lib/brotli/version.rb

@@ -1,3 +1,3 @@
 module Brotli
-  VERSION = '0.7.0'
+  VERSION = '0.9.0'
 end

data/lib/brotli/writer.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module Brotli
+  class Writer
+    def initialize(io, options = nil, **kwargs)
+      raise ArgumentError, "io should not be nil" if io.nil?
+
+      @io = io
+      @compressor = Compressor.new(normalize_options(options, kwargs))
+      @closed = false
+    end
+
+    def write(data)
+      ensure_open
+
+      data = coerce_string(data)
+      write_output(@compressor.process(data))
+      data.bytesize
+    end
+
+    def finish
+      ensure_open
+
+      write_output(@compressor.finish)
+      @io
+    end
+
+    def flush
+      ensure_open
+
+      write_output(@compressor.flush)
+      @io.flush if @io.respond_to?(:flush)
+      self
+    end
+
+    def close
+      return @io if @closed
+
+      begin
+        finish
+      ensure
+        begin
+          @io.close if @io.respond_to?(:close)
+        ensure
+          @closed = true
+          @compressor = nil
+        end
+      end
+
+      @io
+    end
+
+    def closed?
+      @closed
+    end
+
+    private
+
+    def ensure_open
+      raise Error, "Writer is closed" if @closed
+    end
+
+    def write_output(output)
+      @io.write(output) unless output.empty?
+    end
+
+    def normalize_options(options, kwargs)
+      return kwargs if options.nil?
+
+      unless options.is_a?(Hash)
+        raise TypeError, "no implicit conversion of #{options.class} into Hash"
+      end
+
+      kwargs.empty? ? options : options.merge(kwargs)
+    end
+
+    def coerce_string(value)
+      String.try_convert(value) || raise(TypeError, "no implicit conversion of #{value.class} into String")
+    end
+  end
+end

data/lib/brotli.rb

@@ -1,2 +1,4 @@
 require 'brotli/version'
 require 'brotli/brotli'
+require 'brotli/writer'
+require 'brotli/reader'

data/vendor/brotli/c/common/constants.c

@@ -6,10 +6,10 @@
 
 #include "constants.h"
 
-const BrotliPrefixCodeRange
-    _kBrotliPrefixCodeRanges[BROTLI_NUM_BLOCK_LEN_SYMBOLS] = {
-        {1, 2},     {5, 2},     {9, 2},   {13, 2},    {17, 3},    {25, 3},
-        {33, 3},    {41, 3},    {49, 4},  {65, 4},    {81, 4},    {97, 4},
-        {113, 5},   {145, 5},   {177, 5}, {209, 5},   {241, 6},   {305, 6},
-        {369, 7},   {497, 8},   {753, 9}, {1265, 10}, {2289, 11}, {4337, 12},
-        {8433, 13}, {16625, 24}};
+const BROTLI_MODEL("small")
+BrotliPrefixCodeRange _kBrotliPrefixCodeRanges[BROTLI_NUM_BLOCK_LEN_SYMBOLS] = {
+    {1, 2},     {5, 2},     {9, 2},   {13, 2},    {17, 3},    {25, 3},
+    {33, 3},    {41, 3},    {49, 4},  {65, 4},    {81, 4},    {97, 4},
+    {113, 5},   {145, 5},   {177, 5}, {209, 5},   {241, 6},   {305, 6},
+    {369, 7},   {497, 8},   {753, 9}, {1265, 10}, {2289, 11}, {4337, 12},
+    {8433, 13}, {16625, 24}};

data/vendor/brotli/c/common/constants.h

@@ -12,9 +12,6 @@
 #ifndef BROTLI_COMMON_CONSTANTS_H_
 #define BROTLI_COMMON_CONSTANTS_H_
 
-#include <brotli/port.h>
-#include <brotli/types.h>
-
 #include "platform.h"
 
 /* Specification: 7.3. Encoding of the context map */
@@ -195,7 +192,7 @@ typedef struct {
 } BrotliPrefixCodeRange;
 
 /* "Soft-private", it is exported, but not "advertised" as API. */
-BROTLI_COMMON_API extern const BrotliPrefixCodeRange
-    _kBrotliPrefixCodeRanges[BROTLI_NUM_BLOCK_LEN_SYMBOLS];
+BROTLI_COMMON_API extern const BROTLI_MODEL("small")
+BrotliPrefixCodeRange _kBrotliPrefixCodeRanges[BROTLI_NUM_BLOCK_LEN_SYMBOLS];
 
 #endif  /* BROTLI_COMMON_CONSTANTS_H_ */

data/vendor/brotli/c/common/context.c

@@ -1,9 +1,9 @@
 #include "context.h"
 
-#include <brotli/types.h>
+#include "platform.h"
 
 /* Common context lookup table for all context modes. */
-const uint8_t _kBrotliContextLookupTable[2048] = {
+const BROTLI_MODEL("small") uint8_t _kBrotliContextLookupTable[2048] = {
   /* CONTEXT_LSB6, last byte. */
    0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15,
   16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,

data/vendor/brotli/c/common/context.h

@@ -88,8 +88,7 @@
 #ifndef BROTLI_COMMON_CONTEXT_H_
 #define BROTLI_COMMON_CONTEXT_H_
 
-#include <brotli/port.h>
-#include <brotli/types.h>
+#include "platform.h"
 
 typedef enum ContextType {
   CONTEXT_LSB6 = 0,