polyphony 0.87 → 0.89

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 938f929b8373ea094ae045c5acc830f6b848a5018bc5604eab280c740e72f56f
-  data.tar.gz: e0806785ca3b5ea6101640b8bbfba9ece96f5c614e13267e07a1f15c33efeb1f
+  metadata.gz: 9b79252b4e1c8617c398d6804b93063b8d155d1b5d813b65cb2c9a2a1ce4e5e0
+  data.tar.gz: 619e0c77d6ed5bc3d4d0c33e143d5f18eaf83e34d5fc8fe576524a554843faeb
 SHA512:
-  metadata.gz: db4f627ab9fd78899711f8eb12c359e038835ada7d43e494cf0377a2a7f6795746a1e877987f32934e94700cb536c27b58e53d7a5cf77161863a4d26c55abcb5
-  data.tar.gz: f7ac358243a16baa8e83923cf46ac7d8ce422a9e0faeea28faa3e0dd523b6b666180fcf1f5182e86c534ced64cfe680895bcdf7bdc9f16e77b641940f439d992
+  metadata.gz: 137e6a8387305c160af5a59c7d7df91c18e09cd5ba954da3fcd78f339d877aa7d91430ae212852dc8bba5771e12cce4e6fa5b51350ebfb75cc3c61e979babf8c
+  data.tar.gz: 8f0bfeb1b9d05fd7b6494a17f23d79ecb28b825b0e37ace7be913a42eee589859320ebd904313f28e691eef0e5951109acca4be95dfe53156edc108c3dab060f

data/.github/workflows/test_io_uring.yml

@@ -8,15 +8,19 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        ruby: [2.6, 2.7, 3.0]
+        ruby: ['2.7', '3.0', '3.1', 'head']
 
     name: >-
       ${{matrix.os}}, ${{matrix.ruby}}
 
     runs-on: ${{matrix.os}}
     steps:
-    - uses: actions/checkout@v1
-    - uses: ruby/setup-ruby@v1
+    - name: Checkout repository and submodules
+      uses: actions/checkout@v2
+      with:
+        submodules: recursive
+    - name: Setup Ruby
+      uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{matrix.ruby}}
         bundler-cache: true # 'bundle install' and cache
@@ -29,4 +33,4 @@ jobs:
     - name: Compile C-extension
       run: bundle exec rake compile
     - name: Run tests
-      run:  bundle exec rake test
+      run:  bundle exec ruby test/run.rb --verbose --name test_sleep

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 0.89 2022-03-21
+
+- Implement compression/decompression to/from strings (#86)
+
+## 0.88 2022-03-18
+
+- Improve IO stream compression utilities (release GVL, cleanup on exception)
+
 ## 0.87 2022-03-17
 
 - Fix compilation on non-Linux OSes

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    polyphony (0.87)
+    polyphony (0.89)
 
 GEM
   remote: https://rubygems.org/

data/ext/polyphony/backend_common.c

@@ -363,8 +363,6 @@ inline void set_fd_blocking_mode(int fd, int blocking) {
 }
 
 inline void io_verify_blocking_mode(rb_io_t *fptr, VALUE io, VALUE blocking) {
-  int flags;
-  int is_nonblocking;
   VALUE blocking_mode = rb_ivar_get(io, ID_ivar_blocking_mode);
   if (blocking == blocking_mode) return;
 
@@ -481,7 +479,7 @@ struct io_buffer get_io_buffer(VALUE in) {
     struct raw_buffer *raw = FIX2PTR(in);
     return (struct io_buffer){ raw->ptr, raw->len, 1 };
   }
-  return (struct io_buffer){ RSTRING_PTR(in), RSTRING_LEN(in), 0 };
+  return (struct io_buffer){ (unsigned char *)RSTRING_PTR(in), RSTRING_LEN(in), 0 };
 }
 
 VALUE coerce_io_string_or_buffer(VALUE buf) {

data/ext/polyphony/backend_io_uring.c

@@ -262,6 +262,22 @@ inline struct backend_stats backend_get_stats(VALUE self) {
   return backend_base_stats(&backend->base);
 }
 
+static inline struct io_uring_sqe *io_uring_backend_get_sqe(Backend_t *backend) {
+  struct io_uring_sqe *sqe;
+try:
+  sqe = io_uring_get_sqe(&backend->ring);
+  if (sqe) goto done;
+
+  if (backend->pending_sqes)
+    io_uring_backend_immediate_submit(backend);
+  else {
+    VALUE resume_value = backend_snooze(&backend->base);
+    RAISE_IF_EXCEPTION(resume_value);
+  }
+done:
+  return sqe;
+}
+
 VALUE Backend_wakeup(VALUE self) {
   Backend_t *backend;
   GetBackend(self, backend);
@@ -269,7 +285,7 @@ VALUE Backend_wakeup(VALUE self) {
   if (backend->base.currently_polling) {
     // Since we're currently blocking while waiting for a completion, we add a
     // NOP which would cause the io_uring_enter syscall to return
-    struct io_uring_sqe *sqe = io_uring_get_sqe(&backend->ring);
+    struct io_uring_sqe *sqe = io_uring_backend_get_sqe(backend);
     io_uring_prep_nop(sqe);
     io_uring_backend_immediate_submit(backend);
 
@@ -302,7 +318,7 @@ int io_uring_backend_defer_submit_and_await(
 
     // op was not completed (an exception was raised), so we need to cancel it
     ctx->result = -ECANCELED;
-    sqe = io_uring_get_sqe(&backend->ring);
+    sqe = io_uring_backend_get_sqe(backend);
     io_uring_prep_cancel(sqe, (__u64)ctx, 0);
     io_uring_backend_immediate_submit(backend);
   }
@@ -317,7 +333,7 @@ VALUE io_uring_backend_wait_fd(Backend_t *backend, int fd, int write) {
   op_context_t *ctx = context_store_acquire(&backend->store, OP_POLL);
   VALUE resumed_value = Qnil;
 
-  struct io_uring_sqe *sqe = io_uring_get_sqe(&backend->ring);
+  struct io_uring_sqe *sqe = io_uring_backend_get_sqe(backend);
   io_uring_prep_poll_add(sqe, fd, write ? POLLOUT : POLLIN);
 
   io_uring_backend_defer_submit_and_await(backend, sqe, ctx, &resumed_value);
@@ -385,7 +401,7 @@ VALUE Backend_read(VALUE self, VALUE io, VALUE str, VALUE length, VALUE to_eof,
   while (1) {
     VALUE resume_value = Qnil;
     op_context_t *ctx = context_store_acquire(&backend->store, OP_READ);
-    struct io_uring_sqe *sqe = io_uring_get_sqe(&backend->ring);
+    struct io_uring_sqe *sqe = io_uring_backend_get_sqe(backend);
     int result;
     int completed;
     
@@ -453,7 +469,7 @@ VALUE Backend_read_loop(VALUE self, VALUE io, VALUE maxlen) {
   while (1) {
     VALUE resume_value = Qnil;
     op_context_t *ctx = context_store_acquire(&backend->store, OP_READ);
-    struct io_uring_sqe *sqe = io_uring_get_sqe(&backend->ring);
+    struct io_uring_sqe *sqe = io_uring_backend_get_sqe(backend);
     ssize_t result;
     int completed;
 
@@ -502,7 +518,7 @@ VALUE Backend_feed_loop(VALUE self, VALUE io, VALUE receiver, VALUE method) {
   while (1) {
     VALUE resume_value = Qnil;
     op_context_t *ctx = context_store_acquire(&backend->store, OP_READ);
-    struct io_uring_sqe *sqe = io_uring_get_sqe(&backend->ring);
+    struct io_uring_sqe *sqe = io_uring_backend_get_sqe(backend);
     ssize_t result;
     int completed;
 
@@ -546,7 +562,7 @@ VALUE Backend_write(VALUE self, VALUE io, VALUE str) {
   while (left > 0) {
     VALUE resume_value = Qnil;
     op_context_t *ctx = context_store_acquire(&backend->store, OP_WRITE);
-    struct io_uring_sqe *sqe = io_uring_get_sqe(&backend->ring);
+    struct io_uring_sqe *sqe = io_uring_backend_get_sqe(backend);
     int result;
     int completed;
 
@@ -597,7 +613,7 @@ VALUE Backend_writev(VALUE self, VALUE io, int argc, VALUE *argv) {
   while (1) {
     VALUE resume_value = Qnil;
     op_context_t *ctx = context_store_acquire(&backend->store, OP_WRITEV);
-    struct io_uring_sqe *sqe = io_uring_get_sqe(&backend->ring);
+    struct io_uring_sqe *sqe = io_uring_backend_get_sqe(backend);
     int result;
     int completed;
 
@@ -689,7 +705,7 @@ VALUE Backend_recv(VALUE self, VALUE io, VALUE str, VALUE length, VALUE pos) {
   while (1) {
     VALUE resume_value = Qnil;
     op_context_t *ctx = context_store_acquire(&backend->store, OP_RECV);
-    struct io_uring_sqe *sqe = io_uring_get_sqe(&backend->ring);
+    struct io_uring_sqe *sqe = io_uring_backend_get_sqe(backend);
     int result;
     int completed;
 
@@ -739,7 +755,7 @@ VALUE Backend_recv_loop(VALUE self, VALUE io, VALUE maxlen) {
   while (1) {
     VALUE resume_value = Qnil;
     op_context_t *ctx = context_store_acquire(&backend->store, OP_RECV);
-    struct io_uring_sqe *sqe = io_uring_get_sqe(&backend->ring);
+    struct io_uring_sqe *sqe = io_uring_backend_get_sqe(backend);
     int result;
     int completed;
 
@@ -787,7 +803,7 @@ VALUE Backend_recv_feed_loop(VALUE self, VALUE io, VALUE receiver, VALUE method)
   while (1) {
     VALUE resume_value = Qnil;
     op_context_t *ctx = context_store_acquire(&backend->store, OP_RECV);
-    struct io_uring_sqe *sqe = io_uring_get_sqe(&backend->ring);
+    struct io_uring_sqe *sqe = io_uring_backend_get_sqe(backend);
     int result;
     int completed;
 
@@ -831,7 +847,7 @@ VALUE Backend_send(VALUE self, VALUE io, VALUE str, VALUE flags) {
   while (left > 0) {
     VALUE resume_value = Qnil;
     op_context_t *ctx = context_store_acquire(&backend->store, OP_SEND);
-    struct io_uring_sqe *sqe = io_uring_get_sqe(&backend->ring);
+    struct io_uring_sqe *sqe = io_uring_backend_get_sqe(backend);
     int result;
     int completed;
 
@@ -869,7 +885,7 @@ VALUE io_uring_backend_accept(Backend_t *backend, VALUE server_socket, VALUE soc
   while (1) {
     VALUE resume_value = Qnil;
     op_context_t *ctx = context_store_acquire(&backend->store, OP_ACCEPT);
-    struct io_uring_sqe *sqe = io_uring_get_sqe(&backend->ring);
+    struct io_uring_sqe *sqe = io_uring_backend_get_sqe(backend);
     int fd;
     int completed;
 
@@ -935,7 +951,7 @@ VALUE io_uring_backend_splice(Backend_t *backend, VALUE src, VALUE dest, VALUE m
 
   while (1) {
     op_context_t *ctx = context_store_acquire(&backend->store, OP_SPLICE);
-    struct io_uring_sqe *sqe = io_uring_get_sqe(&backend->ring);
+    struct io_uring_sqe *sqe = io_uring_backend_get_sqe(backend);
     int result;
     int completed;
 
@@ -984,7 +1000,7 @@ VALUE Backend_tee(VALUE self, VALUE src, VALUE dest, VALUE maxlen) {
 
   while (1) {
     op_context_t *ctx = context_store_acquire(&backend->store, OP_SPLICE);
-    struct io_uring_sqe *sqe = io_uring_get_sqe(&backend->ring);
+    struct io_uring_sqe *sqe = io_uring_backend_get_sqe(backend);
     int result;
     int completed;
 
@@ -1021,7 +1037,7 @@ VALUE Backend_connect(VALUE self, VALUE sock, VALUE host, VALUE port) {
   GetBackend(self, backend);
   fd = fd_from_io(sock, &fptr, 1, 0);
   ctx = context_store_acquire(&backend->store, OP_CONNECT);
-  sqe = io_uring_get_sqe(&backend->ring);
+  sqe = io_uring_backend_get_sqe(backend);
   io_uring_prep_connect(sqe, fd, ai_addr, ai_addrlen);
   result = io_uring_backend_defer_submit_and_await(backend, sqe, ctx, &resume_value);
   completed = context_store_release(&backend->store, ctx);
@@ -1063,7 +1079,7 @@ VALUE Backend_wait_io(VALUE self, VALUE io, VALUE write) {
 //   io_unset_nonblock(fptr, io);
 
 //   ctx = context_store_acquire(&backend->store, OP_CLOSE);
-//   sqe = io_uring_get_sqe(&backend->ring);
+//   sqe = io_uring_backend_get_sqe(backend);
 //   io_uring_prep_close(sqe, fd);
 //   result = io_uring_backend_defer_submit_and_await(backend, sqe, ctx, &resume_value);
 //   completed = context_store_release(&backend->store, ctx);
@@ -1094,7 +1110,7 @@ inline struct __kernel_timespec duration_to_timespec(VALUE duration) {
 // returns true if completed, 0 otherwise
 int io_uring_backend_submit_timeout_and_await(Backend_t *backend, double duration, VALUE *resume_value) {
   struct __kernel_timespec ts = double_to_timespec(duration);
-  struct io_uring_sqe *sqe = io_uring_get_sqe(&backend->ring);
+  struct io_uring_sqe *sqe = io_uring_backend_get_sqe(backend);
   op_context_t *ctx = context_store_acquire(&backend->store, OP_TIMEOUT);
   
   io_uring_prep_timeout(sqe, &ts, 0, 0);
@@ -1183,7 +1199,7 @@ VALUE Backend_timeout(int argc, VALUE *argv, VALUE self) {
   GetBackend(self, backend);
   timeout = rb_funcall(cTimeoutException, ID_new, 0);
 
-  sqe = io_uring_get_sqe(&backend->ring);
+  sqe = io_uring_backend_get_sqe(backend);
   ctx = context_store_acquire(&backend->store, OP_TIMEOUT);
   ctx->resume_value = timeout;
   io_uring_prep_timeout(sqe, &ts, 0, 0);
@@ -1259,7 +1275,7 @@ VALUE Backend_wait_event(VALUE self, VALUE raise) {
     struct io_uring_sqe *sqe;
 
     backend->event_fd_ctx = context_store_acquire(&backend->store, OP_POLL);
-    sqe = io_uring_get_sqe(&backend->ring);
+    sqe = io_uring_backend_get_sqe(backend);
     io_uring_prep_poll_add(sqe, backend->event_fd, POLLIN);
     backend->base.op_count++;
     io_uring_sqe_set_data(sqe, backend->event_fd_ctx);
@@ -1275,7 +1291,7 @@ VALUE Backend_wait_event(VALUE self, VALUE raise) {
 
     // last fiber to use the eventfd, so we cancel the ongoing poll
     struct io_uring_sqe *sqe;
-    sqe = io_uring_get_sqe(&backend->ring);
+    sqe = io_uring_backend_get_sqe(backend);
     io_uring_prep_cancel(sqe, (__u64)backend->event_fd_ctx, 0);
     io_uring_backend_immediate_submit(backend);
     backend->event_fd_ctx = NULL;
@@ -1296,7 +1312,7 @@ struct io_uring_sqe *Backend_chain_prepare_write(Backend_t *backend, VALUE io, V
   struct io_uring_sqe *sqe;
 
   fd = fd_from_io(io, &fptr, 1, 0);
-  sqe = io_uring_get_sqe(&backend->ring);
+  sqe = io_uring_backend_get_sqe(backend);
   io_uring_prep_write(sqe, fd, StringValuePtr(str), RSTRING_LEN(str), 0);
   return sqe;
 }
@@ -1308,7 +1324,7 @@ struct io_uring_sqe *Backend_chain_prepare_send(Backend_t *backend, VALUE io, VA
 
   fd = fd_from_io(io, &fptr, 1, 0);
 
-  sqe = io_uring_get_sqe(&backend->ring);
+  sqe = io_uring_backend_get_sqe(backend);
   io_uring_prep_send(sqe, fd, StringValuePtr(str), RSTRING_LEN(str), NUM2INT(flags));
   return sqe;
 }
@@ -1322,7 +1338,7 @@ struct io_uring_sqe *Backend_chain_prepare_splice(Backend_t *backend, VALUE src,
 
   src_fd = fd_from_io(src, &src_fptr, 0, 0);
   dest_fd = fd_from_io(dest, &dest_fptr, 1, 0);
-  sqe = io_uring_get_sqe(&backend->ring);
+  sqe = io_uring_backend_get_sqe(backend);
   io_uring_prep_splice(sqe, src_fd, -1, dest_fd, -1, NUM2INT(maxlen), 0);
   return sqe;
 }
@@ -1381,7 +1397,7 @@ VALUE Backend_chain(int argc,VALUE *argv, VALUE self) {
 
         ctx->ref_count = sqe_count;
         ctx->result = -ECANCELED;
-        sqe = io_uring_get_sqe(&backend->ring);
+        sqe = io_uring_backend_get_sqe(backend);
         io_uring_prep_cancel(sqe, (__u64)ctx, 0);
         io_uring_backend_immediate_submit(backend);
       }
@@ -1411,7 +1427,7 @@ VALUE Backend_chain(int argc,VALUE *argv, VALUE self) {
 
     // op was not completed (an exception was raised), so we need to cancel it
     ctx->result = -ECANCELED;
-    sqe = io_uring_get_sqe(&backend->ring);
+    sqe = io_uring_backend_get_sqe(backend);
     io_uring_prep_cancel(sqe, (__u64)ctx, 0);
     io_uring_backend_immediate_submit(backend);
     RAISE_IF_EXCEPTION(resume_value);
@@ -1470,14 +1486,14 @@ static inline void splice_chunks_get_sqe(
   }
   else
     *ctx = context_store_acquire(&backend->store, type);
-  (*sqe) = io_uring_get_sqe(&backend->ring);
+  (*sqe) = io_uring_backend_get_sqe(backend);
 }
 
 static inline void splice_chunks_cancel(Backend_t *backend, op_context_t *ctx) {
   struct io_uring_sqe *sqe;
   
   ctx->result = -ECANCELED;
-  sqe = io_uring_get_sqe(&backend->ring);
+  sqe = io_uring_backend_get_sqe(backend);
   io_uring_prep_cancel(sqe, (__u64)ctx, 0);
   io_uring_backend_immediate_submit(backend);
 }

data/ext/polyphony/extconf.rb

@@ -47,6 +47,7 @@ end
 $defs << '-DPOLYPHONY_USE_PIDFD_OPEN' if config[:pidfd_open]
 if config[:io_uring]
   $defs << "-DPOLYPHONY_BACKEND_LIBURING"
+  $defs << "-DPOLYPHONY_LINUX"
   $defs << "-DPOLYPHONY_UNSET_NONBLOCK" if RUBY_VERSION =~ /^3/
   $CFLAGS << " -Wno-pointer-arith"
 else

data/ext/polyphony/io_extensions.c

@@ -10,6 +10,7 @@
 #include "polyphony.h"
 #include "zlib.h"
 #include "assert.h"
+#include "ruby/thread.h"
 
 ID ID_at;
 ID ID_read_method;
@@ -29,6 +30,7 @@ VALUE SYM_orig_name;
 VALUE SYM_readpartial;
 
 enum read_method {
+  RM_STRING,
   RM_BACKEND_READ,
   RM_BACKEND_RECV,
   RM_READPARTIAL,
@@ -36,13 +38,15 @@ enum read_method {
 };
 
 enum write_method {
+  WM_STRING,
   WM_BACKEND_WRITE,
   WM_BACKEND_SEND,
   WM_WRITE,
   WM_CALL
 };
 
-static enum read_method detect_read_method(VALUE io) {
+static inline enum read_method detect_read_method(VALUE io) {
+  if (TYPE(io) == T_STRING) return RM_STRING;
   if (rb_respond_to(io, ID_read_method)) {
     VALUE method = rb_funcall(io, ID_read_method, 0);
     if (method == SYM_readpartial)  return RM_READPARTIAL;
@@ -58,7 +62,8 @@ static enum read_method detect_read_method(VALUE io) {
     rb_raise(rb_eRuntimeError, "Given io instance should be a callable or respond to #__read_method__");
 }
 
-static enum write_method detect_write_method(VALUE io) {
+static inline enum write_method detect_write_method(VALUE io) {
+  if (TYPE(io) == T_STRING) return WM_STRING;
   if (rb_respond_to(io, ID_write_method)) {
     VALUE method = rb_funcall(io, ID_write_method, 0);
     if (method == SYM_readpartial)    return WM_WRITE;
@@ -136,11 +141,18 @@ static inline int read_to_raw_buffer(VALUE backend, VALUE io, enum read_method m
       RB_GC_GUARD(str);
       return len;
     }
+    default: {
+      rb_raise(rb_eRuntimeError, "Invalid read method");
+    }
   }
 }
 
 static inline int write_from_raw_buffer(VALUE backend, VALUE io, enum write_method method, struct raw_buffer *buffer) {
   switch (method) {
+    case WM_STRING: {
+      rb_str_buf_cat(io, (char *)buffer->ptr, buffer->len);
+      return buffer->len;
+    }
     case WM_BACKEND_WRITE: {
       VALUE len = Backend_write(backend, io, PTR2FIX(buffer));
       return FIX2INT(len);
@@ -165,6 +177,9 @@ static inline int write_from_raw_buffer(VALUE backend, VALUE io, enum write_meth
       RB_GC_GUARD(str);
       return buffer->len;
     }
+    default: {
+      rb_raise(rb_eRuntimeError, "Invalid write method");
+    }
   }
 }
 
@@ -228,7 +243,7 @@ static inline time_t time_from_object(VALUE o) {
   return FIX2INT(rb_funcall(o, rb_intern("to_i"), 0));
 }
 
-int gzip_prepare_header(struct gzip_header_ctx *ctx, char *buffer, int maxlen) {
+int gzip_prepare_header(struct gzip_header_ctx *ctx, unsigned char *buffer, int maxlen) {
   int len = 0;
   unsigned char flags = 0, extraflags = 0;
 
@@ -259,7 +274,7 @@ int gzip_prepare_header(struct gzip_header_ctx *ctx, char *buffer, int maxlen) {
   return len;
 }
 
-int gzip_prepare_footer(unsigned long crc32, unsigned long total_in, char *buffer, int maxlen) {
+static inline int gzip_prepare_footer(unsigned long crc32, unsigned long total_in, unsigned char *buffer, int maxlen) {
   assert(maxlen >= GZIP_FOOTER_LEN);
 
   gzfile_set32(crc32, buffer);
@@ -287,8 +302,8 @@ struct z_stream_ctx {
 
   unsigned char in[CHUNK];
   unsigned char out[CHUNK];
-  int in_pos;
-  int out_pos;
+  unsigned int in_pos;
+  unsigned int out_pos;
   unsigned long in_total;
   unsigned long out_total;
 
@@ -297,8 +312,8 @@ struct z_stream_ctx {
 
 typedef int (*zlib_func)(z_streamp, int);
 
-void read_gzip_header_str(struct raw_buffer *buffer, VALUE *str, int *in_pos, unsigned long *total_read) {
-  int null_pos;
+void read_gzip_header_str(struct raw_buffer *buffer, VALUE *str, unsigned int *in_pos, unsigned long *total_read) {
+  unsigned long null_pos;
   // find null terminator
   for (null_pos = *in_pos; null_pos < *total_read; null_pos++) {
     if (!buffer->ptr[null_pos]) break;
@@ -306,28 +321,37 @@ void read_gzip_header_str(struct raw_buffer *buffer, VALUE *str, int *in_pos, un
   if (null_pos == *total_read)
     rb_raise(rb_eRuntimeError, "Invalid gzip header");
   
-  *str = rb_str_new_cstr(buffer->ptr + *in_pos);
+  *str = rb_str_new_cstr((char *)buffer->ptr + *in_pos);
   *in_pos = null_pos + 1;
 }
 
 void gzip_read_header(struct z_stream_ctx *ctx, struct gzip_header_ctx *header_ctx) {
-  struct raw_buffer in_buffer = { ctx->in, CHUNK };
+  struct raw_buffer in_buffer;
   int flags;
 
-  while (ctx->in_total < 10) {
-    int read = read_to_raw_buffer(ctx->backend, ctx->src, ctx->src_read_method, &in_buffer);
-    if (read == 0) goto error;
-    ctx->in_total += read;
+  if (ctx->src_read_method == RM_STRING) {
+    in_buffer.ptr = (unsigned char *)RSTRING_PTR(ctx->src);
+    in_buffer.len = RSTRING_LEN(ctx->src);
+    ctx->in_total = in_buffer.len;
+  }
+  else {
+    in_buffer.ptr = ctx->in;
+    in_buffer.len = CHUNK;
+    while (ctx->in_total < 10) {
+      int read = read_to_raw_buffer(ctx->backend, ctx->src, ctx->src_read_method, &in_buffer);
+      if (read == 0) goto error;
+      ctx->in_total += read;
+    }
   }
+
   // PRINT_BUFFER("read gzip header", ctx->in, ctx->in_total);
-  if (ctx->in[0] != GZ_MAGIC1) goto error;
-  if (ctx->in[1] != GZ_MAGIC2) goto error;
-  if (ctx->in[2] != GZ_METHOD_DEFLATE) goto error;
-  flags = ctx->in[3];
+  if (in_buffer.ptr[0] != GZ_MAGIC1) goto error;
+  if (in_buffer.ptr[1] != GZ_MAGIC2) goto error;
+  if (in_buffer.ptr[2] != GZ_METHOD_DEFLATE) goto error;
+  flags = in_buffer.ptr[3];
 
-  unsigned long mtime = gzfile_get32(ctx->in + 4);
+  unsigned long mtime = gzfile_get32(in_buffer.ptr + 4);
   header_ctx->mtime = INT2FIX(mtime);
-
   ctx->in_pos = 10;
 
   if (flags & GZ_FLAG_ORIG_NAME)
@@ -344,6 +368,26 @@ error:
   rb_raise(rb_eRuntimeError, "Invalid gzip header");
 }
 
+struct process_z_stream_ctx {
+  z_stream *strm;
+  int flags;
+  zlib_func fun;
+  int ret;
+};
+
+void *do_process_z_stream_without_gvl(void *ptr) {
+  struct process_z_stream_ctx *ctx = (struct process_z_stream_ctx *)ptr;
+
+  ctx->ret = (ctx->fun)(ctx->strm, ctx->flags);
+  return NULL;
+}
+
+static inline int process_without_gvl(zlib_func fun, z_stream *strm, int flags) {
+  struct process_z_stream_ctx ctx = { strm, flags, fun, 0 };
+  rb_thread_call_without_gvl2(do_process_z_stream_without_gvl, (void *)&ctx, RUBY_UBF_IO, 0);
+  return ctx.ret;
+}
+
 static inline int z_stream_write_out(struct z_stream_ctx *ctx, zlib_func fun, int eof) {
   int ret;
   int written;
@@ -351,7 +395,7 @@ static inline int z_stream_write_out(struct z_stream_ctx *ctx, zlib_func fun, in
 
   int avail_out_pre = ctx->strm.avail_out = CHUNK - ctx->out_pos;
   ctx->strm.next_out = ctx->out + ctx->out_pos;
-  ret = fun(&ctx->strm, eof ? Z_FINISH : Z_NO_FLUSH);
+  ret = process_without_gvl(fun, &ctx->strm, eof ? Z_FINISH : Z_NO_FLUSH);
   assert(ret != Z_STREAM_ERROR);
   written = avail_out_pre - ctx->strm.avail_out;
   out_buffer.ptr = ctx->out;
@@ -372,12 +416,13 @@ static inline int z_stream_write_out(struct z_stream_ctx *ctx, zlib_func fun, in
   return ctx->strm.avail_out;
 }
 
-void z_stream_io_loop(struct z_stream_ctx *ctx) {
+VALUE z_stream_io_loop(struct z_stream_ctx *ctx) {
   zlib_func fun = (ctx->mode == SM_DEFLATE) ? deflate : inflate;
 
-  if (ctx->in_total > ctx->in_pos) {
+  if ((ctx->src_read_method != RM_STRING) && (ctx->in_total > ctx->in_pos)) {
     // In bytes already read for parsing gzip header, so we need to process the
     // rest.
+
     ctx->strm.next_in = ctx->in + ctx->in_pos;
     ctx->strm.avail_in = ctx->in_total -= ctx->in_pos;
 
@@ -389,14 +434,27 @@ void z_stream_io_loop(struct z_stream_ctx *ctx) {
   }
 
   while (1) {
-    struct raw_buffer in_buffer = {ctx->in, CHUNK};
-    ctx->strm.next_in = ctx->in;
-    int read_len = ctx->strm.avail_in = read_to_raw_buffer(ctx->backend, ctx->src, ctx->src_read_method, &in_buffer);
-    if (!read_len) break;
-    int eof = read_len < CHUNK;
-
-    if (ctx->mode == SM_DEFLATE)
-      ctx->crc32 = crc32(ctx->crc32, ctx->in, read_len);
+    int eof;
+    int read_len;
+    if (ctx->src_read_method == RM_STRING) {
+      struct raw_buffer in_buffer = {
+        (unsigned char *)RSTRING_PTR(ctx->src) + ctx->in_pos,
+        RSTRING_LEN(ctx->src) - ctx->in_pos
+      };
+      ctx->strm.next_in = in_buffer.ptr;
+      read_len = ctx->strm.avail_in = in_buffer.len;
+      eof = 1;
+      if (ctx->mode == SM_DEFLATE) ctx->crc32 = crc32(ctx->crc32, in_buffer.ptr, read_len);
+    }
+    else {
+      struct raw_buffer in_buffer = {ctx->in, CHUNK};
+      ctx->strm.next_in = ctx->in;
+      read_len = ctx->strm.avail_in = read_to_raw_buffer(ctx->backend, ctx->src, ctx->src_read_method, &in_buffer);
+      if (!read_len) break;
+      eof = read_len < CHUNK;
+      if (ctx->mode == SM_DEFLATE) ctx->crc32 = crc32(ctx->crc32, ctx->in, read_len);
+    }
+
     ctx->in_total += read_len;
 
     // PRINT_BUFFER("read stream", ctx->in, read_len);
@@ -407,13 +465,15 @@ void z_stream_io_loop(struct z_stream_ctx *ctx) {
       if (z_stream_write_out(ctx, fun, eof)) break;
     }
 
-    if (eof) return;
+    if (eof) goto done;
   }
 
   //flush
   ctx->strm.avail_in = 0;
   ctx->strm.next_in = ctx->in;
   z_stream_write_out(ctx, fun, 1);
+done:
+  return Qnil;
 }
 
 static inline void setup_ctx(struct z_stream_ctx *ctx, enum stream_mode mode, VALUE src, VALUE dest) {
@@ -434,6 +494,18 @@ static inline void setup_ctx(struct z_stream_ctx *ctx, enum stream_mode mode, VA
   ctx->crc32 = 0;
 }
 
+static inline VALUE z_stream_cleanup(struct z_stream_ctx *ctx) {
+  if (ctx->mode == SM_DEFLATE)
+    deflateEnd(&ctx->strm);
+  else
+    inflateEnd(&ctx->strm);
+  return Qnil;
+}
+
+#define SAFE(f) (VALUE (*)(VALUE))(f)
+#define Z_STREAM_SAFE_IO_LOOP_WITH_CLEANUP(ctx) \
+  rb_ensure(SAFE(z_stream_io_loop), (VALUE)&ctx, SAFE(z_stream_cleanup), (VALUE)&ctx)
+
 VALUE IO_gzip(int argc, VALUE *argv, VALUE self) {
   VALUE src;
   VALUE dest;
@@ -450,18 +522,16 @@ VALUE IO_gzip(int argc, VALUE *argv, VALUE self) {
   };
 
   struct z_stream_ctx ctx;
-  int level = DEFAULT_LEVEL;
   int ret;
 
   setup_ctx(&ctx, SM_DEFLATE, src, dest);
   ctx.f_gzip_footer = 1; // write gzip footer
-  ctx.out_pos = gzip_prepare_header(&header_ctx, ctx.out, sizeof(ctx.out));
+  ctx.out_total = ctx.out_pos = gzip_prepare_header(&header_ctx, ctx.out, sizeof(ctx.out));
 
-  ret = deflateInit2(&ctx.strm, level, Z_DEFLATED, -MAX_WBITS, DEFAULT_MEM_LEVEL, Z_DEFAULT_STRATEGY);
-  if (ret != Z_OK) return INT2FIX(ret);
-  z_stream_io_loop(&ctx);
-  deflateEnd(&ctx.strm);
- 
+  ret = deflateInit2(&ctx.strm, DEFAULT_LEVEL, Z_DEFLATED, -MAX_WBITS, DEFAULT_MEM_LEVEL, Z_DEFAULT_STRATEGY);
+  if (ret != Z_OK)
+    rb_raise(rb_eRuntimeError, "zlib error: %s\n", ctx.strm.msg);
+  Z_STREAM_SAFE_IO_LOOP_WITH_CLEANUP(ctx); 
   return INT2FIX(ctx.out_total);
 }
 
@@ -482,21 +552,25 @@ VALUE IO_gunzip(int argc, VALUE *argv, VALUE self) {
   setup_ctx(&ctx, SM_INFLATE, src, dest);
   gzip_read_header(&ctx, &header_ctx);
 
+  ret = inflateInit2(&ctx.strm, -MAX_WBITS);
+  if (ret != Z_OK)
+    rb_raise(rb_eRuntimeError, "zlib error: %s\n", ctx.strm.msg);
+
+  Z_STREAM_SAFE_IO_LOOP_WITH_CLEANUP(ctx);
+
+  // gzip_read_footer(&ctx, &footer_ctx);
+  // TODO: verify crc32
+  // TODO: verify total length
+
   if (info != Qnil) {
     rb_hash_aset(info, SYM_mtime, FIX2TIME(header_ctx.mtime));
     rb_hash_aset(info, SYM_orig_name, header_ctx.orig_name);
     rb_hash_aset(info, SYM_comment, header_ctx.comment);
   }
+  RB_GC_GUARD(header_ctx.orig_name);
+  RB_GC_GUARD(header_ctx.comment);
 
-  ret = inflateInit2(&ctx.strm, -MAX_WBITS);
-  if (ret != Z_OK) return INT2FIX(ret);
-  z_stream_io_loop(&ctx);
-  inflateEnd(&ctx.strm);
-
-  // gzip_read_footer(&ctx, &footer_ctx);
-  // TODO: verify crc32
-  // TODO: verify total length
-  return self;
+  return INT2FIX(ctx.out_total);
 }
 
 VALUE IO_deflate(VALUE self, VALUE src, VALUE dest) {
@@ -506,9 +580,10 @@ VALUE IO_deflate(VALUE self, VALUE src, VALUE dest) {
 
   setup_ctx(&ctx, SM_DEFLATE, src, dest);
   ret = deflateInit(&ctx.strm, level);
-  if (ret != Z_OK) return INT2FIX(ret);
-  z_stream_io_loop(&ctx);
-  deflateEnd(&ctx.strm);
+  if (ret != Z_OK)
+    rb_raise(rb_eRuntimeError, "zlib error: %s\n", ctx.strm.msg);
+
+  Z_STREAM_SAFE_IO_LOOP_WITH_CLEANUP(ctx);
  
   return INT2FIX(ctx.out_total);
 }
@@ -519,9 +594,10 @@ VALUE IO_inflate(VALUE self, VALUE src, VALUE dest) {
 
   setup_ctx(&ctx, SM_INFLATE, src, dest);
   ret = inflateInit(&ctx.strm);
-  if (ret != Z_OK) return INT2FIX(ret);
-  z_stream_io_loop(&ctx);
-  inflateEnd(&ctx.strm);
+  if (ret != Z_OK)
+    rb_raise(rb_eRuntimeError, "zlib error: %s\n", ctx.strm.msg);
+
+  Z_STREAM_SAFE_IO_LOOP_WITH_CLEANUP(ctx);
  
   return INT2FIX(ctx.out_total);
 }
@@ -538,16 +614,16 @@ VALUE IO_http1_splice_chunked(VALUE self, VALUE src, VALUE dest, VALUE maxlen) {
     if (!len) break;
 
     // write chunk header
-    buffer.len += sprintf(buffer.ptr + buffer.len, "%x\r\n", len);
+    buffer.len += sprintf((char *)buffer.ptr + buffer.len, "%x\r\n", len);
     write_from_raw_buffer(backend, dest, method, &buffer);
     buffer.len = 0;
     while (len) {
       int spliced = FIX2INT(Backend_splice(backend, pipe, dest, INT2FIX(len)));
       len -= spliced;
     }
-    buffer.len += sprintf(buffer.ptr + buffer.len, "\r\n");
+    buffer.len += sprintf((char *)buffer.ptr + buffer.len, "\r\n");
   }
-  buffer.len += sprintf(buffer.ptr + buffer.len, "0\r\n\r\n");
+  buffer.len += sprintf((char *)buffer.ptr + buffer.len, "0\r\n\r\n");
   write_from_raw_buffer(backend, dest, method, &buffer);
 
   Pipe_close(pipe);

data/ext/polyphony/polyphony.c

@@ -145,7 +145,7 @@ VALUE Polyphony_raw_buffer_get(int argc, VALUE *argv, VALUE self) {
   int length = (len == Qnil) ? buffer->len : FIX2INT(len);
   
   if (length > buffer->len) length = buffer->len;
-  return rb_utf8_str_new(buffer->ptr, length);
+  return rb_utf8_str_new((char *)buffer->ptr, length);
 }
 
 VALUE Polyphony_raw_buffer_set(VALUE self, VALUE buf, VALUE str) {
@@ -164,12 +164,6 @@ VALUE Polyphony_raw_buffer_size(VALUE self, VALUE buf) {
   return INT2FIX(buffer->len);
 }
 
-VALUE Polyphony_backend_test(VALUE self, VALUE io, VALUE str) {
-  struct raw_buffer buffer = { RSTRING_PTR(str), RSTRING_LEN(str) };
-  VALUE args[2] = { io, PTR2FIX(&buffer) };
-  return Polyphony_backend_write(2, args, self);
-}
-
 // VALUE Polyphony_backend_close(VALUE self, VALUE io) {
 //   return Backend_close(BACKEND(), io);
 // }
@@ -210,7 +204,6 @@ void Init_Polyphony() {
   rb_define_singleton_method(mPolyphony, "__raw_buffer_get__", Polyphony_raw_buffer_get, -1);
   rb_define_singleton_method(mPolyphony, "__raw_buffer_set__", Polyphony_raw_buffer_set, 2);
   rb_define_singleton_method(mPolyphony, "__raw_buffer_size__", Polyphony_raw_buffer_size, 1);
-  rb_define_singleton_method(mPolyphony, "backend_test", Polyphony_backend_test, 2);
 
   rb_define_global_function("snooze", Polyphony_snooze, 0);
   rb_define_global_function("suspend", Polyphony_suspend, 0);

data/lib/polyphony/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Polyphony
-  VERSION = '0.87'
+  VERSION = '0.89'
 end

data/test/test_io.rb

@@ -656,19 +656,75 @@ class IOExtensionsTest < MiniTest::Test
     i, o = IO.pipe
     r, w = IO.pipe
 
-    spin {
-      IO.deflate(i, w)
+    ret = nil
+    f = spin {
+      ret = IO.deflate(i, w)
       w.close
     }
 
     o << 'foobar' * 20
     o.close
 
+    f.await
+    assert_equal 17, ret
+
     data = r.read
     msg = Zlib::Inflate.inflate(data)
     assert_equal 'foobar' * 20, msg
   end
 
+  def test_deflate_to_string
+    i, o = IO.pipe
+    r, w = IO.pipe
+    str = +''
+
+    ret = nil
+    f = spin {
+      ret = IO.deflate(i, str)
+      w << str
+      w.close
+    }
+
+    o << 'foobar' * 20
+    o.close
+
+    f.await
+    assert_equal 17, ret
+
+    data = r.read
+    msg = Zlib::Inflate.inflate(data)
+    assert_equal 'foobar' * 20, msg
+  end
+
+  def test_deflate_to_frozen_string
+    i, o = IO.pipe
+    str = '' # frozen
+
+    f = spin {
+      o << 'foobar' * 20
+      o.close
+    }
+
+    assert_raises(FrozenError) { IO.deflate(i, str) }
+  end
+
+  def test_deflate_from_string
+    r, w = IO.pipe
+    str = 'foobar' * 10000
+    ret = nil
+    
+    f = spin {
+      ret = IO.deflate(str, w)
+      w.close
+    }
+    f.await
+    assert_equal 118, ret
+
+    data = r.read
+    msg = Zlib::Inflate.inflate(data)
+    assert_equal str, msg
+  end
+
   def test_inflate
     i, o = IO.pipe
     r, w = IO.pipe
@@ -679,7 +735,35 @@ class IOExtensionsTest < MiniTest::Test
       o.close
     }
 
-    IO.inflate(i, w)
+    ret = IO.inflate(i, w)
+    assert_equal 6, ret
+    w.close
+    msg = r.read
+    assert_equal 'foobar', msg
+  end
+
+  def test_inflate_to_string
+    i, o = IO.pipe
+    str = +''
+
+    spin {
+      data = Zlib::Deflate.deflate('foobar', 9)
+      o << data
+      o.close
+    }
+
+    ret = IO.inflate(i, str)
+    assert_equal 6, ret
+    assert_equal 6, str.bytesize
+    assert_equal 'foobar', str
+  end
+
+  def test_inflate_from_string
+    r, w = IO.pipe
+    str = Zlib::Deflate.deflate('foobar', 9)
+
+    ret = IO.inflate(str, w)
+    assert_equal 6, ret
     w.close
     msg = r.read
     assert_equal 'foobar', msg
@@ -690,7 +774,7 @@ class IOExtensionsTest < MiniTest::Test
     dest = Polyphony.pipe
     now = nil
 
-    spin {
+    f = spin {
       now = Time.now
       IO.gzip(src, dest)
       dest.close
@@ -698,6 +782,32 @@ class IOExtensionsTest < MiniTest::Test
 
     src << IO.read(__FILE__)
     src.close
+    f.await
+
+    gz = Zlib::GzipReader.new(dest)
+    data = gz.read
+    assert_equal IO.read(__FILE__), data
+    assert_in_range (now-2)..(now+1), gz.mtime
+    assert_nil gz.orig_name
+    assert_nil gz.comment
+  end
+
+  def test_gzip_to_string
+    src = Polyphony.pipe
+    dest = Polyphony.pipe
+    str = +''
+    now = nil
+
+    f = spin {
+      now = Time.now
+      IO.gzip(src, str)
+      dest << str
+      dest.close
+    }
+
+    src << IO.read(__FILE__)
+    src.close
+    f.await
 
     gz = Zlib::GzipReader.new(dest)
     data = gz.read
@@ -707,6 +817,39 @@ class IOExtensionsTest < MiniTest::Test
     assert_nil gz.comment
   end
 
+  def test_gzip_from_string
+    str = IO.read(__FILE__)
+    dest = Polyphony.pipe
+    now = nil
+
+    IO.gzip(str, dest)
+    dest.close
+
+    gz = Zlib::GzipReader.new(dest)
+    data = gz.read
+    assert_equal IO.read(__FILE__), data
+  end
+
+  def test_gzip_return_value
+    src = Polyphony.pipe
+    dest = Polyphony.pipe
+    now = nil
+    ret = nil
+
+    f = spin {
+      now = Time.now
+      ret = IO.gzip(src, dest)
+      dest.close
+    }
+
+    src << IO.read(__FILE__)
+    src.close
+    f.await
+
+    gzipped = dest.read
+    assert_equal gzipped.bytesize, ret
+  end
+
   def test_gzip_with_mtime_int
     src = Polyphony.pipe
     dest = Polyphony.pipe
@@ -801,19 +944,51 @@ class IOExtensionsTest < MiniTest::Test
   def test_gunzip
     src = Polyphony.pipe
     dest = Polyphony.pipe
+    ret = nil
 
-    spin {
-      IO.gunzip(src, dest)
+    f = spin {
+      ret = IO.gunzip(src, dest)
       dest.close
     }
 
     gz = Zlib::GzipWriter.new(src, 9)
-    gz << 'foobar'#IO.read(__FILE__)
+    gz << IO.read(__FILE__)
     gz.close
+    f.await
 
     data = dest.read
-    # assert_equal IO.read(__FILE__), data
-    assert_equal 'foobar', data
+    assert_equal IO.read(__FILE__).bytesize, ret
+    assert_equal IO.read(__FILE__), data
+  end
+
+  def test_gunzip_to_string
+    src = Polyphony.pipe
+    str = +''
+    ret = nil
+
+    f = spin {
+      ret = IO.gunzip(src, str)
+    }
+
+    gz = Zlib::GzipWriter.new(src, 9)
+    gz << IO.read(__FILE__)
+    gz.close
+    f.await
+
+    assert_equal IO.read(__FILE__).bytesize, ret
+    assert_equal IO.read(__FILE__), str
+  end
+
+  def test_gunzip_from_string
+    src_data = 'foobar' * 1000
+    str = Zlib.gzip(src_data, level: 9)
+    dest = Polyphony.pipe
+    ret = IO.gunzip(str, dest)
+    dest.close
+
+    dest_data = dest.read
+    assert_equal src_data.bytesize, ret
+    assert_equal src_data, dest_data
   end
 
   def test_gunzip_multi
@@ -893,4 +1068,24 @@ class IOExtensionsTest < MiniTest::Test
     assert_equal 'foo.bar', dest_info[:orig_name]
     assert_equal 'hello!', dest_info[:comment]
   end
+
+  def test_deflate_inflate_strings
+    src_data = IO.read(__FILE__)
+    deflated = +''
+    IO.deflate(src_data, deflated)
+    inflated = +''
+    IO.inflate(deflated, inflated)
+
+    assert_equal src_data, inflated
+  end
+
+  def test_gzip_gunzip_strings
+    src_data = IO.read(__FILE__)
+    gzipped = +''
+    IO.gzip(src_data, gzipped)
+    gunzipped = +''
+    IO.gunzip(gzipped, gunzipped)
+
+    assert_equal src_data, gunzipped
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: polyphony
 version: !ruby/object:Gem::Version
-  version: '0.87'
+  version: '0.89'
 platform: ruby
 authors:
 - Sharon Rosner
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-03-17 00:00:00.000000000 Z
+date: 2022-03-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake-compiler
@@ -136,7 +136,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.1.4
-description: 
+description:
 email: sharon@noteflakes.com
 executables: []
 extensions:
@@ -639,7 +639,7 @@ metadata:
   documentation_uri: https://digital-fabric.github.io/polyphony/
   homepage_uri: https://digital-fabric.github.io/polyphony/
   changelog_uri: https://github.com/digital-fabric/polyphony/blob/master/CHANGELOG.md
-post_install_message: 
+post_install_message:
 rdoc_options:
 - "--title"
 - polyphony
@@ -658,8 +658,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
-signing_key: 
+rubygems_version: 3.3.3
+signing_key:
 specification_version: 4
 summary: Fine grained concurrency for Ruby
 test_files: []