ruby-lzws 1.1.4 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6fc8861bd91fe484c375cdeae3ad55e478727e8a1314117fb3897cef408bd401
-  data.tar.gz: 9b7a0d9cd62930a12250641599638d80abb34737bae20842109d91f1dfee84a8
+  metadata.gz: 46de1059ebc7e81a3c93869a70801e20571e99631a0a9a8acc0cad9b06e9c21f
+  data.tar.gz: c4d1d3859bd374404b496b9ae393f34b08b49bf94655e420862545fd8949cc12
 SHA512:
-  metadata.gz: 880eb17f48cca872b77a56936aedebe4aa416eb971e51743550e0bd02c473568fb638d412deab64c522fa84895732c9f91da338a7d4f79bd3ac4c382f35326eb
-  data.tar.gz: e29e72c646f05e55ec4411a06e4be7122581962d9fe46164bc6bbaeefd1e874595ad74783503f76bf049096730043032976c8595e2d7903d9acb3ab9bb39b8cc
+  metadata.gz: 88f2f3627fb6f98fffc6c1289301a9362c6ab22edc7de42732a90c85b1065bd2e3ee73547dbab0ee1c40d14c3490053f43d36d7b67abb25aa6ec2e938211e95e
+  data.tar.gz: e6b97fea5d39bd2ee8b4e9ad43d16453f0640536c86dd9e0d425443c3a48a5bb3bf99c5bb452e4065f3c21489572a4bf05a9809b1eccfd7ef39773a1eea0d05d

data/README.md

@@ -1,14 +1,14 @@
 # Ruby bindings for lzws library
 
-| Travis | AppVeyor | Cirrus | Circle | Codecov |
-| :---:  | :---:    | :---:  | :---:  | :---:   |
-| [![Travis test status](https://travis-ci.com/andrew-aladev/ruby-lzws.svg?branch=master)](https://travis-ci.com/andrew-aladev/ruby-lzws) | [![AppVeyor test status](https://ci.appveyor.com/api/projects/status/github/andrew-aladev/ruby-lzws?branch=master&svg=true)](https://ci.appveyor.com/project/andrew-aladev/ruby-lzws/branch/master) | [![Cirrus test status](https://api.cirrus-ci.com/github/andrew-aladev/ruby-lzws.svg?branch=master)](https://cirrus-ci.com/github/andrew-aladev/ruby-lzws) | [![Circle test status](https://circleci.com/gh/andrew-aladev/ruby-lzws/tree/master.svg?style=shield)](https://circleci.com/gh/andrew-aladev/ruby-lzws/tree/master) | [![Codecov](https://codecov.io/gh/andrew-aladev/ruby-lzws/branch/master/graph/badge.svg)](https://codecov.io/gh/andrew-aladev/ruby-lzws) |
+| Travis | AppVeyor | Circle | Codecov | Gem   |
+| :---:  | :---:    | :---:  | :---:   | :---: |
+| [![Travis test status](https://travis-ci.com/andrew-aladev/ruby-lzws.svg?branch=master)](https://travis-ci.com/andrew-aladev/ruby-lzws) | [![AppVeyor test status](https://ci.appveyor.com/api/projects/status/github/andrew-aladev/ruby-lzws?branch=master&svg=true)](https://ci.appveyor.com/project/andrew-aladev/ruby-lzws/branch/master) | [![Circle test status](https://circleci.com/gh/andrew-aladev/ruby-lzws/tree/master.svg?style=shield)](https://circleci.com/gh/andrew-aladev/ruby-lzws/tree/master) | [![Codecov](https://codecov.io/gh/andrew-aladev/ruby-lzws/branch/master/graph/badge.svg)](https://codecov.io/gh/andrew-aladev/ruby-lzws) | [![Gem](https://img.shields.io/gem/v/ruby-lzws.svg)](https://rubygems.org/gems/ruby-lzws) |
 
 See [lzws library](https://github.com/andrew-aladev/lzws).
 
 ## Installation
 
-Please install lzws library first, use latest 1.3.0+ version.
+Please install lzws library first, use latest 1.4.0+ version.
 
 ```sh
 gem install ruby-lzws
@@ -21,6 +21,8 @@ rake gem
 gem install pkg/ruby-lzws-*.gem
 ```
 
+You can also use [overlay](https://github.com/andrew-aladev/overlay) for gentoo.
+
 ## Usage
 
 There are simple APIs: `String` and `File`. Also you can use generic streaming API: `Stream::Writer` and `Stream::Reader`.
@@ -36,10 +38,31 @@ LZWS::File.decompress "file.txt.Z", "file.txt"
 
 LZWS::Stream::Writer.open("file.txt.Z") { |writer| writer << "TOBEORNOTTOBEORTOBEORNOT" }
 puts LZWS::Stream::Reader.open("file.txt.Z") { |reader| reader.read }
+
+writer = LZWS::Stream::Writer.new output_socket
+begin
+  bytes_written = writer.write_nonblock "TOBEORNOTTOBEORTOBEORNOT"
+  # handle "bytes_written"
+rescue IO::WaitWritable
+  # handle wait
+ensure
+  writer.close
+end
+
+reader = LZWS::Stream::Reader.new input_socket
+begin
+  puts reader.read_nonblock(512)
+rescue IO::WaitReadable
+  # handle wait
+rescue ::EOFError
+  # handle eof
+ensure
+  reader.close
+end
 ```
 
-You can create and read `tar.Z` archives with `minitar` for example.
-LZWS is compatible with UNIX compress (with default options).
+You can create and read `tar.Z` archives with [minitar](https://github.com/halostatue/minitar) for example.
+LZWS is compatible with [UNIX compress](https://en.wikipedia.org/wiki/Compress) (with default options).
 
 ```ruby
 require "lzws"
@@ -61,66 +84,51 @@ LZWS::Stream::Reader.open "file.tar.Z" do |reader|
 end
 ```
 
-## Options
-
-Each API supports several options:
-
-```
-:source_buffer_length
-:destination_buffer_length
-```
-
-There are internal buffers for compressed and decompressed data.
-For example you want to use 1 KB as source buffer length for compressor - please use 256 B as destination buffer length.
-You want to use 256 B as source buffer length for decompressor - please use 1 KB as destination buffer length.
-
-Values: 0, 2 - infinity, default value: 0.
-0 means automatic buffer length selection.
-1 byte is not enough, 2 bytes is minimal buffer length.
+You can also use `Content-Encoding: compress` with [sinatra](http://sinatrarb.com):
 
-```
-:max_code_bit_length
-```
-
-Values: `LZWS::Option::LOWEST_MAX_CODE_BIT_LENGTH` - `LZWS::Option::BIGGEST_MAX_CODE_BIT_LENGTH`, default value: `LZWS::Option::BIGGEST_MAX_CODE_BIT_LENGTH`.
-
-```
-:block_mode
-```
-
-Values: true/false, default value: true.
-
-```
-:without_magic_header
-```
-
-Values: true/false, default value: false.
+```ruby
+require "lzws"
+require "sinatra"
 
-```
-:msb
+get "/" do
+  headers["Content-Encoding"] = "compress"
+  LZWS::String.compress "TOBEORNOTTOBEORTOBEORNOT"
+end
 ```
 
-Values: true/false, default value: false.
+## Options
 
-```
-:unaligned_bit_groups
-```
+| Option                      | Values     | Default  | Description |
+|-----------------------------|------------|----------|-------------|
+| `source_buffer_length`      | 0, 2 - inf | 0 (auto) | internal buffer length for source data |
+| `destination_buffer_length` | 0, 2 - inf | 0 (auto) | internal buffer length for description data |
+| `gvl`                       | true/false | false    | enables global VM lock where possible |
+| `max_code_bit_length`       | 9 - 16     | 16       | max code bit length |
+| `block_mode`                | true/false | true     | enables block mode |
+| `without_magic_header`      | true/false | false    | disables magic header |
+| `msb`                       | true/false | false    | enables most significant bit mode |
+| `unaligned_bit_groups`      | true/false | false    | enables unaligned bit groups |
+| `quiet`                     | true/false | false    | disables lzws library logging |
 
-Values: true/false, default value: false.
+There are internal buffers for compressed and decompressed data.
+For example you want to use 1 KB as `source_buffer_length` for compressor - please use 256 B as `destination_buffer_length`.
+You want to use 256 B as `source_buffer_length` for decompressor - please use 1 KB as `destination_buffer_length`.
 
-```
-:quiet
-```
+`gvl` is disabled by default, this mode allows running multiple compressors/decompressors in different threads simultaneously.
+Please consider enabling `gvl` if you don't want to launch processors in separate threads.
+If `gvl` is enabled ruby won't waste time on acquiring/releasing VM lock.
 
-Values: true/false, default value: false.
-Disables lzws library logging.
+You can also read lzws docs for more info about options.
 
-Please read lzws docs for more info about options.
+| Option                | Related constants |
+|-----------------------|-------------------|
+| `max_code_bit_length` | `LZWS::Option::LOWEST_MAX_CODE_BIT_LENGTH` = 9, `LZWS::Option::BIGGEST_MAX_CODE_BIT_LENGTH` = 16 |
 
 Possible compressor options:
 ```
 :source_buffer_length
 :destination_buffer_length
+:gvl
 :max_code_bit_length
 :block_mode
 :without_magic_header
@@ -133,6 +141,7 @@ Possible decompressor options:
 ```
 :source_buffer_length
 :destination_buffer_length
+:gvl
 :without_magic_header
 :msb
 :unaligned_bit_groups
@@ -148,18 +157,6 @@ data = LZWS::String.compress "TOBEORNOTTOBEORTOBEORNOT", :msb => true
 puts LZWS::String.decompress(data, :msb => true)
 ```
 
-Default options are compatible with UNIX compress (`Content-Encoding: compress`):
-
-```ruby
-require "lzws"
-require "sinatra"
-
-get "/" do
-  headers["Content-Encoding"] = "compress"
-  LZWS::String.compress "TOBEORNOTTOBEORTOBEORNOT"
-end
-```
-
 Please read more about compatibility in lzws docs.
 
 ## String
@@ -333,14 +330,18 @@ See [`IO`](https://ruby-doc.org/core-2.7.0/IO.html) docs.
 
 Typical helpers, see [`Zlib::GzipReader`](https://ruby-doc.org/stdlib-2.7.0/libdoc/zlib/rdoc/Zlib/GzipReader.html) docs.
 
+## Thread safety
+
+`:gvl` option is disabled by default, you can use bindings effectively in multiple threads.
+Please be careful: bindings are not thread safe.
+You should lock all shared data between threads.
+
 ## CI
 
-See universal test script [scripts/ci_test.sh](scripts/ci_test.sh) for CI.
 Please visit [scripts/test-images](scripts/test-images).
-You can run this test script using many native and cross images.
-
-Cirrus CI uses `x86_64-pc-linux-gnu` image, Circle CI - `x86_64-gentoo-linux-musl` image.
+See universal test script [scripts/ci_test.sh](scripts/ci_test.sh) for CI.
+You can run this script using many native and cross images.
 
 ## License
 
-MIT license, see LICENSE and AUTHORS.
+MIT license, see [LICENSE](LICENSE) and [AUTHORS](AUTHORS).

data/ext/extconf.rb

@@ -3,6 +3,8 @@
 
 require "mkmf"
 
+have_func "rb_thread_call_without_gvl", "ruby/thread.h"
+
 def require_header(name, types = [])
   abort "Can't find #{name} header" unless find_header name
 
@@ -13,10 +15,10 @@ end
 
 require_header "lzws/buffer.h"
 require_header "lzws/common.h", %w[lzws_result_t]
-require_header "lzws/compressor/common.h"
+require_header "lzws/compressor/common.h", %w[lzws_compressor_options_t]
 require_header "lzws/compressor/main.h"
 require_header "lzws/compressor/state.h", %w[lzws_compressor_state_t]
-require_header "lzws/decompressor/common.h"
+require_header "lzws/decompressor/common.h", %w[lzws_decompressor_options_t]
 require_header "lzws/decompressor/main.h"
 require_header "lzws/decompressor/state.h", %w[lzws_decompressor_state_t]
 require_header "lzws/file.h"
@@ -66,6 +68,13 @@ $srcs = %w[
 .map { |name| "src/#{extension_name}/#{name}.c" }
 .freeze
 
+# Removing library duplicates.
+$libs = $libs.split(%r{\s})
+  .reject(&:empty?)
+  .sort
+  .uniq
+  .join " "
+
 if ENV["CI"] || ENV["COVERAGE"]
   $CFLAGS << " --coverage"
   $LDFLAGS << " --coverage"

data/ext/lzws_ext/buffer.c

@@ -12,10 +12,11 @@ VALUE lzws_ext_create_string_buffer(VALUE length)
   return rb_str_new(NULL, NUM2SIZET(length));
 }
 
-VALUE lzws_ext_resize_string_buffer(VALUE args)
+VALUE lzws_ext_resize_string_buffer(VALUE buffer_args)
 {
-  VALUE buffer = rb_ary_entry(args, 0);
-  VALUE length = rb_ary_entry(args, 1);
+  VALUE buffer = rb_ary_entry(buffer_args, 0);
+  VALUE length = rb_ary_entry(buffer_args, 1);
+
   return rb_str_resize(buffer, NUM2SIZET(length));
 }
 
@@ -23,8 +24,21 @@ void lzws_ext_buffer_exports(VALUE root_module)
 {
   VALUE module = rb_define_module_under(root_module, "Buffer");
 
-  rb_define_const(module, "DEFAULT_SOURCE_BUFFER_LENGTH_FOR_COMPRESSOR", SIZET2NUM(LZWS_DEFAULT_SOURCE_BUFFER_LENGTH_FOR_COMPRESSOR));
-  rb_define_const(module, "DEFAULT_DESTINATION_BUFFER_LENGTH_FOR_COMPRESSOR", SIZET2NUM(LZWS_DEFAULT_DESTINATION_BUFFER_LENGTH_FOR_COMPRESSOR));
-  rb_define_const(module, "DEFAULT_SOURCE_BUFFER_LENGTH_FOR_DECOMPRESSOR", SIZET2NUM(LZWS_DEFAULT_SOURCE_BUFFER_LENGTH_FOR_DECOMPRESSOR));
-  rb_define_const(module, "DEFAULT_DESTINATION_BUFFER_LENGTH_FOR_DECOMPRESSOR", SIZET2NUM(LZWS_DEFAULT_DESTINATION_BUFFER_LENGTH_FOR_DECOMPRESSOR));
+  rb_define_const(
+    module, "DEFAULT_SOURCE_BUFFER_LENGTH_FOR_COMPRESSOR", SIZET2NUM(LZWS_DEFAULT_SOURCE_BUFFER_LENGTH_FOR_COMPRESSOR));
+
+  rb_define_const(
+    module,
+    "DEFAULT_DESTINATION_BUFFER_LENGTH_FOR_COMPRESSOR",
+    SIZET2NUM(LZWS_DEFAULT_DESTINATION_BUFFER_LENGTH_FOR_COMPRESSOR));
+
+  rb_define_const(
+    module,
+    "DEFAULT_SOURCE_BUFFER_LENGTH_FOR_DECOMPRESSOR",
+    SIZET2NUM(LZWS_DEFAULT_SOURCE_BUFFER_LENGTH_FOR_DECOMPRESSOR));
+
+  rb_define_const(
+    module,
+    "DEFAULT_DESTINATION_BUFFER_LENGTH_FOR_DECOMPRESSOR",
+    SIZET2NUM(LZWS_DEFAULT_DESTINATION_BUFFER_LENGTH_FOR_DECOMPRESSOR));
 }

data/ext/lzws_ext/buffer.h

@@ -11,12 +11,12 @@ VALUE lzws_ext_create_string_buffer(VALUE length);
 #define LZWS_EXT_CREATE_STRING_BUFFER(buffer, length, exception) \
   VALUE buffer = rb_protect(lzws_ext_create_string_buffer, SIZET2NUM(length), &exception);
 
-VALUE lzws_ext_resize_string_buffer(VALUE args);
+VALUE lzws_ext_resize_string_buffer(VALUE buffer_args);
 
-#define LZWS_EXT_RESIZE_STRING_BUFFER(buffer, length, exception)            \
-  VALUE args = rb_ary_new_from_args(2, buffer, SIZET2NUM(length));          \
-  buffer     = rb_protect(lzws_ext_resize_string_buffer, args, &exception); \
-  RB_GC_GUARD(args);
+#define LZWS_EXT_RESIZE_STRING_BUFFER(buffer, length, exception)                          \
+  VALUE buffer_args = rb_ary_new_from_args(2, buffer, SIZET2NUM(length));                 \
+  buffer            = rb_protect(lzws_ext_resize_string_buffer, buffer_args, &exception); \
+  RB_GC_GUARD(buffer_args);
 
 void lzws_ext_buffer_exports(VALUE root_module);
 

data/ext/lzws_ext/error.h

@@ -9,7 +9,8 @@
 
 // Results for errors listed in "lib/lzws/error" used in c extension.
 
-enum {
+enum
+{
   LZWS_EXT_ERROR_ALLOCATE_FAILED = 1,
   LZWS_EXT_ERROR_VALIDATE_FAILED,
 

data/ext/lzws_ext/gvl.h

@@ -0,0 +1,24 @@
+// Ruby bindings for lzws library.
+// Copyright (c) 2019 AUTHORS, MIT License.
+
+#if !defined(LZWS_EXT_GVL_H)
+#define LZWS_EXT_GVL_H
+
+#ifdef HAVE_RB_THREAD_CALL_WITHOUT_GVL
+
+#include "ruby/thread.h"
+
+#define LZWS_EXT_GVL_WRAP(gvl, function, data)                             \
+  if (gvl) {                                                               \
+    function((void*) data);                                                \
+  } else {                                                                 \
+    rb_thread_call_without_gvl(function, (void*) data, RUBY_UBF_IO, NULL); \
+  }
+
+#else
+
+#define LZWS_EXT_GVL_WRAP(_gvl, function, data) function((void*) data);
+
+#endif
+
+#endif // LZWS_EXT_GVL_H

data/ext/lzws_ext/io.c

@@ -1,16 +1,18 @@
 // Ruby bindings for lzws library.
 // Copyright (c) 2019 AUTHORS, MIT License.
 
-#include "ruby/io.h"
+#include "lzws_ext/io.h"
 
-#include <lzws/common.h>
 #include <lzws/file.h>
 
 #include "lzws_ext/error.h"
-#include "lzws_ext/io.h"
+#include "lzws_ext/gvl.h"
 #include "lzws_ext/macro.h"
 #include "lzws_ext/option.h"
 #include "ruby.h"
+#include "ruby/io.h"
+
+// -- utils --
 
 #define GET_FILE(target)                               \
   Check_Type(target, T_FILE);                          \
@@ -48,22 +50,52 @@ static inline lzws_ext_result_t get_file_error(lzws_result_t result)
   }
 }
 
+// -- compress --
+
+typedef struct
+{
+  FILE*                      source_file;
+  size_t                     source_buffer_length;
+  FILE*                      destination_file;
+  size_t                     destination_buffer_length;
+  lzws_compressor_options_t* compressor_options_ptr;
+  lzws_result_t              result;
+} compress_args_t;
+
+static inline void* compress_wrapper(void* data)
+{
+  compress_args_t* args = data;
+
+  args->result = lzws_compress_file(
+    args->source_file,
+    args->source_buffer_length,
+    args->destination_file,
+    args->destination_buffer_length,
+    args->compressor_options_ptr);
+
+  return NULL;
+}
+
 VALUE lzws_ext_compress_io(VALUE LZWS_EXT_UNUSED(self), VALUE source, VALUE destination, VALUE options)
 {
   GET_FILE(source);
   GET_FILE(destination);
   Check_Type(options, T_HASH);
+  LZWS_EXT_GET_SIZE_OPTION(options, source_buffer_length);
+  LZWS_EXT_GET_SIZE_OPTION(options, destination_buffer_length);
+  LZWS_EXT_GET_BOOL_OPTION(options, gvl);
   LZWS_EXT_GET_COMPRESSOR_OPTIONS(options);
-  LZWS_EXT_GET_BUFFER_LENGTH_OPTION(options, source_buffer_length);
-  LZWS_EXT_GET_BUFFER_LENGTH_OPTION(options, destination_buffer_length);
 
-  lzws_result_t result = lzws_compress_file(
-    source_file, source_buffer_length,
-    destination_file, destination_buffer_length,
-    without_magic_header, max_code_bit_length, block_mode, msb, unaligned_bit_groups, quiet);
+  compress_args_t args = {
+    .source_file               = source_file,
+    .source_buffer_length      = source_buffer_length,
+    .destination_file          = destination_file,
+    .destination_buffer_length = destination_buffer_length,
+    .compressor_options_ptr    = &compressor_options};
 
-  if (result != 0) {
-    lzws_ext_raise_error(get_file_error(result));
+  LZWS_EXT_GVL_WRAP(gvl, compress_wrapper, &args);
+  if (args.result != 0) {
+    lzws_ext_raise_error(get_file_error(args.result));
   }
 
   // Ruby itself won't flush stdio file before closing fd, flush is required.
@@ -72,22 +104,52 @@ VALUE lzws_ext_compress_io(VALUE LZWS_EXT_UNUSED(self), VALUE source, VALUE dest
   return Qnil;
 }
 
+// -- decompress --
+
+typedef struct
+{
+  FILE*                        source_file;
+  size_t                       source_buffer_length;
+  FILE*                        destination_file;
+  size_t                       destination_buffer_length;
+  lzws_decompressor_options_t* decompressor_options_ptr;
+  lzws_result_t                result;
+} decompress_args_t;
+
+static inline void* decompress_wrapper(void* data)
+{
+  decompress_args_t* args = data;
+
+  args->result = lzws_decompress_file(
+    args->source_file,
+    args->source_buffer_length,
+    args->destination_file,
+    args->destination_buffer_length,
+    args->decompressor_options_ptr);
+
+  return NULL;
+}
+
 VALUE lzws_ext_decompress_io(VALUE LZWS_EXT_UNUSED(self), VALUE source, VALUE destination, VALUE options)
 {
   GET_FILE(source);
   GET_FILE(destination);
   Check_Type(options, T_HASH);
+  LZWS_EXT_GET_SIZE_OPTION(options, source_buffer_length);
+  LZWS_EXT_GET_SIZE_OPTION(options, destination_buffer_length);
+  LZWS_EXT_GET_BOOL_OPTION(options, gvl);
   LZWS_EXT_GET_DECOMPRESSOR_OPTIONS(options);
-  LZWS_EXT_GET_BUFFER_LENGTH_OPTION(options, source_buffer_length);
-  LZWS_EXT_GET_BUFFER_LENGTH_OPTION(options, destination_buffer_length);
 
-  lzws_result_t result = lzws_decompress_file(
-    source_file, source_buffer_length,
-    destination_file, destination_buffer_length,
-    without_magic_header, msb, unaligned_bit_groups, quiet);
+  decompress_args_t args = {
+    .source_file               = source_file,
+    .source_buffer_length      = source_buffer_length,
+    .destination_file          = destination_file,
+    .destination_buffer_length = destination_buffer_length,
+    .decompressor_options_ptr  = &decompressor_options};
 
-  if (result != 0) {
-    lzws_ext_raise_error(get_file_error(result));
+  LZWS_EXT_GVL_WRAP(gvl, decompress_wrapper, &args);
+  if (args.result != 0) {
+    lzws_ext_raise_error(get_file_error(args.result));
   }
 
   // Ruby itself won't flush stdio file before closing fd, flush is required.
@@ -96,6 +158,8 @@ VALUE lzws_ext_decompress_io(VALUE LZWS_EXT_UNUSED(self), VALUE source, VALUE de
   return Qnil;
 }
 
+// -- exports --
+
 void lzws_ext_io_exports(VALUE root_module)
 {
   rb_define_module_function(root_module, "_native_compress_io", RUBY_METHOD_FUNC(lzws_ext_compress_io), 3);