extbzip3 0.0.1

data/ext/extbzip3_encoder.c

@@ -0,0 +1,344 @@
+#include "extbzip3.h"
+
+static inline int
+aux_oneshot_encode(int format, uint32_t blocksize, const void *in, void *out, size_t insize, size_t *outsize)
+{
+    if (blocksize < AUX_BZIP3_BLOCKSIZE_MIN) {
+        blocksize = AUX_BZIP3_BLOCKSIZE_MIN;
+    } else if (blocksize > AUX_BZIP3_BLOCKSIZE_MAX) {
+        return BZ3_ERR_INIT;
+    }
+
+    if (!outsize) {
+        return BZ3_ERR_INIT;
+    }
+
+    int headersize;
+    size_t blockcount;
+    if (format == AUX_BZIP3_V1_FILE_FORMAT) {
+        headersize = 9;
+        blockcount = 0;
+    } else {
+        headersize = 13;
+        blockcount = (insize / blocksize) + ((insize % blocksize != 0) ? 1 : 0);
+
+        if (blockcount > (UINT32_MAX - 1)) {
+            return BZ3_ERR_DATA_TOO_BIG;
+        }
+    }
+
+    if (*outsize < (size_t)headersize) {
+        return BZ3_ERR_INIT;
+    }
+
+    struct bz3_state *bz3 = bz3_new(blocksize);
+    if (bz3 == NULL) {
+        return BZ3_ERR_INIT;
+    }
+
+    const uint8_t *inp = (const uint8_t *)in;
+    const uint8_t *const inend = inp + insize;
+    uint8_t *outp = (uint8_t *)out + headersize;
+    uint8_t *const outend = outp + *outsize;
+
+    while (inend - inp > 0) {
+        uint32_t origsize = ((inend - inp) > blocksize) ? blocksize : (uint32_t)(inend - inp);
+        uint32_t packedsize = (uint32_t)bz3_bound(origsize); // TODO???: 過剰な値かも？
+
+        if (outend - outp < packedsize) {
+            bz3_free(bz3);
+            return BZ3_ERR_DATA_TOO_BIG;
+        }
+
+        outp += 8;
+
+        memmove(outp, inp, origsize);
+        int32_t ret = aux_bz3_encode_block_nogvl(bz3, outp, origsize);
+        if (ret < 0) {
+            bz3_free(bz3);
+            return ret;
+        }
+
+        storeu32le(outp - 8, ret);
+        storeu32le(outp - 4, origsize);
+
+        inp += origsize;
+        outp += ret;
+    }
+
+    bz3_free(bz3);
+
+    *outsize = (size_t)(outp - (const uint8_t *)out);
+
+    memmove(out, aux_bzip3_signature, sizeof(aux_bzip3_signature));
+    storeu32le((char *)out + 5, blocksize);
+
+    if (format == AUX_BZIP3_V1_FRAME_FORMAT) {
+        storeu32le((char *)out + 9, (uint32_t)blockcount);
+    }
+
+    return BZ3_OK;
+}
+
+struct encoder
+{
+    struct bz3_state *bzip3;
+    uint32_t blocksize;
+    int firstwrite:1;
+    int closed:1;
+    VALUE outport;
+    VALUE srcbuf;
+    VALUE destbuf;
+};
+
+#define ENCODER_FREE_BLOCK(P)                                           \
+        if ((P)->bzip3) {                                               \
+            bz3_free((P)->bzip3);                                       \
+        }                                                               \
+
+#define ENCODER_VALUE_FOREACH(DEF)                                      \
+        DEF(outport)                                                    \
+        DEF(srcbuf)                                                     \
+        DEF(destbuf)                                                    \
+
+AUX_DEFINE_TYPED_DATA(encoder, encoder_allocate, ENCODER_FREE_BLOCK, ENCODER_VALUE_FOREACH)
+
+/*
+ *  @overload initialize(blocksize: (16 << 20))
+ */
+static VALUE
+encoder_initialize(int argc, VALUE argv[], VALUE self)
+{
+    struct { VALUE outport, opts; } args;
+    rb_scan_args(argc, argv, "1:", &args.outport, &args.opts);
+
+    enum { numkw = 1 };
+    ID idtab[numkw] = { rb_intern("blocksize") };
+    union { struct { VALUE blocksize; }; VALUE vect[numkw]; } opts;
+    rb_get_kwargs(args.opts, idtab, 0, numkw, opts.vect);
+
+    struct encoder *p = (struct encoder *)rb_check_typeddata(self, &encoder_type);
+    if (p == NULL || p->bzip3) {
+        rb_raise(rb_eTypeError, "wrong initialized or re-initializing - %" PRIsVALUE, self);
+    }
+
+    p->blocksize = aux_conv_to_blocksize(opts.blocksize);
+    p->outport = args.outport;
+    p->srcbuf = Qnil;
+    p->destbuf = Qnil;
+    p->bzip3 = aux_bz3_new(p->blocksize);
+    p->firstwrite = 1;
+
+    return self;
+}
+
+static VALUE
+encoder_outport(VALUE self)
+{
+    return get_encoder(self)->outport;
+}
+
+static void
+encoder_write_encode(VALUE self, struct encoder *p, const void *buf, size_t len)
+{
+    size_t bufoff = 8 + (p->firstwrite ? 9 : 0);
+
+    p->destbuf = aux_str_new_recycle(p->destbuf, bufoff + bz3_bound(len));
+    rb_str_set_len(p->destbuf, bufoff);
+    rb_str_cat(p->destbuf, buf, len);
+
+    int32_t res = aux_bz3_encode_block_nogvl(p->bzip3, RSTRING_PTR(p->destbuf) + bufoff, RSTRING_LEN(p->destbuf) - bufoff);
+
+    if (res < 0) {
+        extbzip3_check_error(res);
+    }
+
+    rb_str_set_len(p->destbuf, bufoff + res);
+    storeu32le(RSTRING_PTR(p->destbuf) + bufoff - 8, res);
+    storeu32le(RSTRING_PTR(p->destbuf) + bufoff - 4, (uint32_t)len);
+
+    if (p->firstwrite) {
+        memcpy(RSTRING_PTR(p->destbuf), "BZ3v1", 5);
+        storeu32le(RSTRING_PTR(p->destbuf) + 5, p->blocksize);
+        p->firstwrite = 0;
+    }
+
+    rb_funcallv(p->outport, rb_intern("<<"), 1, &p->destbuf);
+}
+
+static VALUE
+encoder_write(VALUE self, VALUE src)
+{
+    struct encoder *p = get_encoder(self);
+
+    rb_check_type(src, RUBY_T_STRING);
+
+    size_t srclen = RSTRING_LEN(src);
+
+    if (srclen <= p->blocksize) {
+        if (rb_type_p(p->srcbuf, RUBY_T_STRING)) {
+            size_t srcbuflen = RSTRING_LEN(p->srcbuf);
+            size_t catlen = srclen + srcbuflen;
+
+            if (catlen >= p->blocksize) {
+                rb_str_cat(p->srcbuf, RSTRING_PTR(src), p->blocksize - srcbuflen);
+                encoder_write_encode(self, p, RSTRING_PTR(p->srcbuf), RSTRING_LEN(p->srcbuf));
+                rb_str_set_len(p->srcbuf, 0);
+                rb_str_cat(p->srcbuf, RSTRING_PTR(src) + p->blocksize - srcbuflen, srclen - (p->blocksize - srcbuflen));
+
+                return self;
+            }
+        } else {
+            p->srcbuf = rb_str_new(0, 0);
+        }
+
+        rb_str_cat(p->srcbuf, RSTRING_PTR(src), srclen);
+
+        return self;
+    } else {
+        size_t srcoff = 0;
+
+        if (rb_type_p(p->srcbuf, RUBY_T_STRING) && RSTRING_LEN(p->srcbuf) > 0) {
+            srcoff = p->blocksize - RSTRING_LEN(p->srcbuf);
+            rb_str_cat(p->srcbuf, RSTRING_PTR(src), srcoff);
+            encoder_write_encode(self, p, RSTRING_PTR(p->srcbuf), RSTRING_LEN(p->srcbuf));
+            rb_str_set_len(p->srcbuf, 0);
+            srclen = RSTRING_LEN(src); // maybe changed src with `outport << destbuf`
+            if (srclen < srcoff) {
+                return self;
+            }
+        }
+
+        while (srclen - srcoff > p->blocksize) {
+            encoder_write_encode(self, p, RSTRING_PTR(src) + srcoff, p->blocksize);
+            srcoff += p->blocksize;
+            srclen = RSTRING_LEN(src); // maybe changed src with `outport << destbuf`
+            if (srclen < srcoff) {
+                return self;
+            }
+        }
+
+        if (srcoff < srclen) {
+            if (!rb_type_p(p->srcbuf, RUBY_T_STRING)) {
+                p->srcbuf = rb_str_new(0, 0);
+            }
+
+            rb_str_cat(p->srcbuf, RSTRING_PTR(src), srclen - srcoff);
+        }
+
+        return self;
+    }
+}
+
+static VALUE
+encoder_flush(VALUE self)
+{
+    struct encoder *p = get_encoder(self);
+
+    if (rb_type_p(p->srcbuf, RUBY_T_STRING) && RSTRING_LEN(p->srcbuf) > 0) {
+        encoder_write_encode(self, p, RSTRING_PTR(p->srcbuf), RSTRING_LEN(p->srcbuf));
+        rb_str_set_len(p->srcbuf, 0);
+    }
+
+    return Qnil;
+}
+
+static VALUE
+encoder_close(VALUE self)
+{
+    struct encoder *p = get_encoder(self);
+
+    if (rb_type_p(p->srcbuf, RUBY_T_STRING) && RSTRING_LEN(p->srcbuf) > 0) {
+        encoder_write_encode(self, p, RSTRING_PTR(p->srcbuf), RSTRING_LEN(p->srcbuf));
+        rb_str_set_len(p->srcbuf, 0);
+    }
+
+    p->closed = 1;
+
+    return Qnil;
+}
+
+static VALUE
+encoder_closed_p(VALUE self)
+{
+    return get_encoder(self)->closed ? Qtrue : Qfalse;
+}
+
+/*
+ *  @overload encode(src, maxdest = nil, dest = "", **opts)
+ *  @overload encode(src, dest, **opts)
+ *
+ *  @return [String]
+ *      dest as decompression data
+ *  @param  [String]    src
+ *  @param  [Integer]   maxdest
+ *  @param  [String]    dest
+ *  @option opts        [Integer]       :blocksize ((16 << 20))
+ *  @option opts                        :format (Bzip3::V1_FILE_FORMAT)
+ *      Bzip3::V1_FILE_FORMAT, Bzip3::V1_FRAME_FORMAT
+ */
+static VALUE
+encoder_s_encode(int argc, VALUE argv[], VALUE mod)
+{
+    size_t insize, outsize;
+    struct { VALUE src, maxdest, dest, opts; } args;
+    switch (rb_scan_args(argc, argv, "12:", &args.src, &args.maxdest, &args.dest, &args.opts)) {
+    case 1:
+        insize = RSTRING_LEN(args.src);
+        outsize = bz3_bound(insize);
+        args.dest = rb_str_buf_new(outsize);
+        break;
+    case 2:
+        insize = RSTRING_LEN(args.src);
+
+        if (rb_type_p(args.maxdest, RUBY_T_FIXNUM) || rb_type_p(args.maxdest, RUBY_T_BIGNUM)) {
+            outsize = NUM2SIZET(args.maxdest);
+            args.dest = rb_str_buf_new(outsize);
+        } else {
+            args.dest = args.maxdest;
+            outsize = bz3_bound(insize);
+            rb_str_modify(args.dest);
+            rb_str_set_len(args.dest, 0);
+            rb_str_modify_expand(args.dest, outsize);
+        }
+
+        break;
+    case 3:
+        insize = RSTRING_LEN(args.src);
+        outsize = NUM2SIZET(args.maxdest);
+        rb_str_modify(args.dest);
+        rb_str_set_len(args.dest, 0);
+        rb_str_modify_expand(args.dest, outsize);
+
+        break;
+    }
+
+    enum { numkw = 2 };
+    ID idtab[numkw] = { rb_intern("blocksize"), rb_intern("format") };
+    union { struct { VALUE blocksize, format; }; VALUE vect[numkw]; } opts;
+    rb_get_kwargs(args.opts, idtab, 0, numkw, opts.vect);
+
+    uint32_t blocksize = aux_conv_to_blocksize(opts.blocksize);
+    int status = aux_oneshot_encode(aux_conv_to_format(opts.format), blocksize,
+                                    RSTRING_PTR(args.src), RSTRING_PTR(args.dest),
+                                    insize, &outsize);
+    extbzip3_check_error(status);
+    rb_str_set_len(args.dest, outsize);
+
+    return args.dest;
+}
+
+void
+extbzip3_init_encoder(VALUE bzip3_module)
+{
+    VALUE encoder_class = rb_define_class_under(bzip3_module, "Encoder", rb_cObject);
+    rb_define_alloc_func(encoder_class, encoder_allocate);
+    rb_define_singleton_method(encoder_class, "encode", encoder_s_encode, -1);
+    rb_define_method(encoder_class, "initialize", encoder_initialize, -1);
+    rb_define_method(encoder_class, "outport", encoder_outport, 0);
+    rb_define_method(encoder_class, "write", encoder_write, 1);
+    rb_define_method(encoder_class, "flush", encoder_flush, 0);
+    rb_define_method(encoder_class, "close", encoder_close, 0);
+    rb_define_method(encoder_class, "closed?", encoder_closed_p, 0);
+    rb_define_alias(encoder_class, "<<", "write");
+}

data/ext/extconf.rb

@@ -0,0 +1,43 @@
+#!ruby
+
+require "mkmf"
+
+using Module.new {
+  refine Object do
+    def has_function_modifier?(modifier_code)
+      try_compile(<<~CODE)
+        #{modifier_code}
+        void
+        func1(void) {
+        }
+      CODE
+    end
+  end
+}
+
+MakeMakefile::CONFIG["optflags"] = "-O0 -g3" if ENV["RUBY_EXTBZIP3_DEBUG"].to_i > 0
+
+have_header("libbz3.h") or abort "need libbz3.h header file"
+have_library("bzip3") or abort "need libbzip3 library"
+
+if RbConfig::CONFIG["arch"] =~ /mingw/i
+  #$LDFLAGS << " -static-libgcc" if try_ldflags("-static-libgcc")
+else
+  if try_compile(<<~"VISIBILITY")
+      __attribute__((visibility("hidden"))) int conftest(void) { return 0; }
+     VISIBILITY
+
+    if try_cflags("-fvisibility=hidden")
+      $CFLAGS << " -fvisibility=hidden"
+      $defs << %(-DEXTBZIP3_API='__attribute__((visibility("default")))')
+    end
+  end
+end
+
+mod = %w(__attribute__((__noreturn__)) __declspec(noreturn) [[noreturn]] _Noreturn).find { |m|
+  has_function_modifier?(m)
+}
+$defs << %(-DRBEXT_NORETURN='#{mod}')
+
+create_header
+create_makefile File.join(RUBY_VERSION.slice(/\d+\.\d+/), "extbzip3")

data/gemstub.rb

@@ -0,0 +1,17 @@
+require_relative "lib/extbzip3/version"
+
+GEMSTUB = Gem::Specification.new do |s|
+  s.name = "extbzip3"
+  s.version = Bzip3::VERSION
+  s.summary = "ruby bindings for bzip3 (bz3)"
+  s.description = <<~DESCRIPT
+    unoficial ruby bindings for bzip3 (bz3) <https://github.com/kspalaiologos/bzip3>.
+  DESCRIPT
+  s.homepage = "https://github.com/dearblue/ruby-extbzip3"
+  s.licenses = %w(BSD-2-Clause LGPL-3.0+ Apache-2.0)
+  s.author = "dearblue"
+  s.email = "dearblue@users.osdn.me"
+
+  #s.required_ruby_version = ">= 2.0"
+  s.add_development_dependency "rake", "~> 0"
+end

data/lib/extbzip3/decoder.rb

@@ -0,0 +1,90 @@
+module Bzip3
+  remove_const :Decoder rescue nil
+
+  class Decoder
+    def initialize(inport, blocksize = 16 << 20)
+      @bzip3 = BlockProcessor.new(blocksize)
+      @inport = inport
+      @readbuf = nil
+      @destbuf = nil
+      @closed = false
+      @eof = false
+
+      self
+    end
+
+    def read(size = nil, buf = "".b)
+      buf ||= "".b
+      buf[0..-1] = "".b
+
+      return buf if size == 0
+
+      size &&= 0 | Integer(0 + size)
+
+      @readbuf ||= "".b
+
+      while true
+        if @destbuf
+          if size && @destbuf.bytesize >= size
+            buf << @destbuf.byteslice(0, size)
+            @destbuf[0...size] = "".b
+            break
+          else
+            buf << @destbuf
+            size -= @destbuf.bytesize if size
+            @destbuf[0..-1] = "".b
+          end
+        else
+          @destbuf = "".b
+
+          (sig, bs) = @inport.read(9)&.unpack("a5V")
+          raise "意図しない EOF" unless bs
+          raise "シグネチャが違う" unless sig == "BZ3v1"
+          raise "おかしなブロックサイズ" unless ((65 << 10) .. (511 << 20)).include? bs
+          raise "ブロックサイズがでかすぎ" unless bs <= @bzip3.blocksize
+        end
+
+        unless b1 = @inport.read(8)
+          @eof = true
+          break
+        end
+        raise "意図しない EOF" if b1.bytesize < 8
+        if b1.byteslice(0, 5) == "BZ3v1"
+          b2 = @inport.read(1)
+          if b2 && b2.bytesize > 0
+            b1 << b2
+            bs = b1.unpack1("@5V")
+            raise "ブロックサイズがでかすぎ" unless bs <= @bzip3.blocksize
+            next
+          end
+        end
+        (packedsize, originsize) = b1.unpack("VV")
+        raise "元のデータがデカすぎ" unless originsize <= @bzip3.blocksize
+        raise "意図しない EOF" unless @inport.read(packedsize, @readbuf)
+        raise "大きさがあってない" unless @readbuf.bytesize == packedsize
+        @bzip3.decode(@readbuf, @destbuf, originsize)
+      end
+
+      buf.empty? ? nil : buf
+    end
+
+    def close
+      @closed = true
+      @bzip3 = nil
+      @readbuf = nil
+      @destbuf = nil
+
+      self
+    end
+
+    def closed?
+      @closed
+    end
+
+    def eof?
+      @eof
+    end
+
+    alias :eof :eof?
+  end
+end

data/lib/extbzip3/fordoc.rb

@@ -0,0 +1,38 @@
+# This file is for additional documents
+
+0 / 0
+
+#
+# Bzip3 module.
+#
+module Bzip3
+  #
+  # Decoder class.
+  #
+  class Decoder
+  end
+
+  #
+  # Encoder class.
+  #
+  class Encoder
+  end
+end
+
+class BasicObject
+  # 自身を出力ストリームと見立てて bzip3 を行うエンコーダオブジェクトを生成します。
+  def to_bzip3
+  end
+
+  # 自身を入力ストリームと見立てて bunzip3 を行うデコーダオブジェクトを生成します。
+  def bunzip3
+  end
+end
+
+class String
+  def to_bzip3
+  end
+
+  def bunzip3
+  end
+end

data/lib/extbzip3/version.rb

@@ -0,0 +1,8 @@
+module Bzip3
+  verreg = /^\s*[\-\*]\s+version:\s*(\d+(?:\.\w+)+)\s*$/i
+  unless File.read("#{__dir__}/../../README.md", mode: "rt") =~ verreg
+    raise "``version'' is not defined or bad syntax in ``README.md''"
+  end
+
+  VERSION = String($1)
+end

data/lib/extbzip3.rb

@@ -0,0 +1,65 @@
+ver = RUBY_VERSION.slice(/\d+\.\d+/)
+soname = File.basename(__FILE__, ".rb") << ".so"
+require File.join(ver, soname)
+
+module Bzip3
+  refine String do
+    def to_bzip3(*args, **opts)
+      Bzip3::Encoder.encode(self, *args, **opts)
+    end
+
+    def bunzip3(*args, **opts)
+      Bzip3::Decoder.decode(self, *args, **opts)
+    end
+  end
+
+  refine BasicObject do
+    def to_bzip3(*args, **opts, &block)
+      Bzip3::Encoder.open(self, *args, **opts, &block)
+    end
+
+    def bunzip3(*args, **opts, &block)
+      Bzip3::Decoder.open(self, *args, **opts, &block)
+    end
+  end
+
+  class << Bzip3
+    using Bzip3
+
+    def encode(src, *args, **opts, &block)
+      src.to_bzip3(*args, **opts, &block)
+    end
+
+    def decode(src, *args, **opts, &block)
+      src.bunzip3(*args, **opts, &block)
+    end
+  end
+
+  class << Decoder
+    def open(*args, **opts, &block)
+      bz3 = new(*args, **opts)
+      return bz3 unless block
+
+      begin
+        yield bz3
+      ensure
+        bz3.close unless bz3.closed?
+      end
+    end
+  end
+
+  class << Encoder
+    def open(*args, **opts, &block)
+      bz3 = new(*args, **opts)
+      return bz3 unless block
+
+      begin
+        yield bz3
+      ensure
+        bz3.close unless bz3.closed?
+      end
+    end
+  end
+
+  Bzip3 = self
+end

data/test/test_bzip3.rb

@@ -0,0 +1,77 @@
+#!ruby
+
+require "test-unit"
+require "extbzip3"
+
+SAMPLES = File.join(__dir__, "../sampledata")
+
+class << SAMPLES
+  def load_file(path)
+    File.binread File.join(self, path)
+  end
+
+  def open_file(path, *args, **opts, &block)
+    File.open File.join(self, path), *args, **opts, &block
+  end
+end
+
+SAMPLES.freeze
+
+class TestBzip3 < Test::Unit::TestCase
+  def test_oneshot
+    assert_equal "123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ\n",
+                 Bzip3.decode(SAMPLES.load_file("single.bz3"))
+
+    assert_raise RuntimeError do
+      Bzip3.decode(SAMPLES.load_file("single.bz3-frame"))
+    end
+
+    assert_raise RuntimeError do
+      Bzip3.decode(SAMPLES.load_file("single.bz3-frame"), format: Bzip3::V1_FILE_FORMAT)
+    end
+
+    assert_equal "123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ\nZYXWVUTSRQPONMLKJIHGFEDCBA987654321\n",
+                 Bzip3.decode(SAMPLES.load_file("double.bz3"))
+
+    assert_equal "123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ\n",
+                 Bzip3.decode(SAMPLES.load_file("double.bz3"), concat: false)
+
+    assert_raise RuntimeError do
+      Bzip3.decode(SAMPLES.load_file("single+junks.bz3"))
+    end
+  end
+
+  def test_oneshot_frame
+    assert_equal "123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ\n",
+                 Bzip3.decode(SAMPLES.load_file("single.bz3-frame"), format: Bzip3::V1_FRAME_FORMAT)
+
+    assert_raise RuntimeError do
+      Bzip3.decode(SAMPLES.load_file("single.bz3"), format: Bzip3::V1_FRAME_FORMAT)
+    end
+  end
+
+  def test_stream
+    io = SAMPLES.open_file("single.bz3")
+    assert_kind_of Bzip3::Decoder, Bzip3.decode(io)
+    assert_equal "123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ\n", Bzip3.decode(io).read
+    io.rewind
+    assert_equal "OK",
+                 Bzip3.decode(io) { |bz3|
+                   assert_equal "", bz3.read(0, "")
+                   assert_equal "12345", bz3.read(5, "")
+                   assert_equal "6789AB", bz3.read(6)
+                   assert_equal "", bz3.read(0)
+                   assert_equal "CDEFGHIJKLMNOPQRSTUVWXYZ\n", bz3.read
+                   assert_equal "", bz3.read(0)
+                   assert_equal nil, bz3.read
+                   "OK"
+                 }
+
+    if defined? Ractor
+      assert_equal "123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ\n", Ractor.new {
+        io2 = SAMPLES.open_file("single.bz3")
+        Bzip3.decode(io2).read
+      }.take
+    end
+  end
+end