iodine 0.7.20 → 0.7.21

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 69bfcfdbbca3218b3e1865afa22a9ccfcbeb8cbbe5eadacbcac978b3bb11d1ec
-  data.tar.gz: 474d23c9adfd030ef247fb695630ea7a81337170f173aa25ac898689605dcaef
+  metadata.gz: b0f9360224da57145ba3ed9e81d5e5472cf9bb9732c9d36fab6e22c32428013a
+  data.tar.gz: e04fc9d016bbb53cf80d810765a5b934d0d996d447faadd6451c27612cd17da6
 SHA512:
-  metadata.gz: b30cb2e9178c39a10458af0368733d2b35dda7f2d5b3a95870ee3147f782a29fd4270358606c7be8459a8924f383bfcb382feb45f58f59bab181fdc158728ca0
-  data.tar.gz: 1ced424fdbc203a0e8efc9739b6a9cd99b3f4bf616f482f677ff9a6c87a336a9c453563d8338f3fc908ed50150699fd1d9bb5a11de4eb69da93869403685791b
+  metadata.gz: 970307f1349dd8ca00a6caa6bfb59c6485f05fdb0b1654dd2788f3b477f628aa9b1c24a9df0873409930dd071dc446e999164731892fe7d70a64983f49a72d99
+  data.tar.gz: aa2417372b97f415ee83edd7aa0ecd557cef4144eba546cd931bc36078574cc03b574f16a395a779957015084c1737b996d13337516ac474efa78d4140075035

data/CHANGELOG.md

@@ -6,9 +6,17 @@ Please notice that this change log contains changes for upcoming releases as wel
 
 ## Changes:
 
+#### Change log v.0.7.21
+
+**Fix**: (`iodine`, `redis`) Redis response was attempting to create Ruby objects outside the GIL. This is now fixed by entering the GIL earlier (before objects are created). Credit to @moxgeek (Marouane Elmidaoui) for exposing this issue (plezi#31).
+
+**Fix**: (`redis`) fixed Redis reconnection. Address and port data was mistakingly written at the wrong address, causing it to be overwritten by incoming (non-pub/sub) data.
+
+**Fix**: (`redis`) fixed a race condition in the Redis reconnection logic which might have caused more then a single pub/sub connection to be established and the first pending command to be sent again.
+
 #### Change log v.0.7.20
 
-**Security**: (`fio`) lower Slowloris detection limits (backlog limit is now 1,024 responses / messages per client). 
+**Security**: (`fio`) lower and smarter Slowloris detection limits (backlog limit is now 1,024 responses / messages per client). 
 
 **Security**: (`http`) HTTP/1.1 slow client throttling - new requests will not be consumed until pending responses were sent. Since HTTP/1.1 is a response-request protocol, this protocol specific approach should protect the HTTP application against slow clients.
 

data/examples/shootout.ru

@@ -32,7 +32,7 @@ module ShootoutApp
   end
   def self.on_message client, data
     if data[0] == 'b' # binary
-        client.publish :shootout_b, data
+      client.publish :shootout_b, data
       data[0] = 'r'
       client.write data
       return

data/ext/iodine/fio.c

@@ -2901,42 +2901,32 @@ void fio_force_close(intptr_t uuid) {
  * error or when the connection is closed.
  */
 ssize_t fio_flush(intptr_t uuid) {
-  if (!uuid_is_valid(uuid)) {
-    errno = EBADF;
-    return -1;
-  }
-  ssize_t flushed;
+  if (!uuid_is_valid(uuid))
+    goto invalid;
+  errno = 0;
+  ssize_t flushed = 0;
   int tmp;
   /* start critical section */
   if (fio_trylock(&uuid_data(uuid).sock_lock))
     goto would_block;
 
-  if (uuid_data(uuid).packet) {
-    tmp = uuid_data(uuid).packet->write_func(fio_uuid2fd(uuid),
-                                             uuid_data(uuid).packet);
-    const size_t old_count = uuid_data(uuid).packet_count;
-    if (tmp == 0) {
-      errno = ECONNRESET;
-      fio_unlock(&uuid_data(uuid).sock_lock);
-      uuid_data(uuid).close = 1;
-      goto closed;
-    } else if (tmp < 0) {
-      goto test_errno;
-    }
+  if (!uuid_data(uuid).packet)
+    goto flush_rw_hook;
 
-    if (old_count >= 1024 && uuid_data(uuid).packet_count == old_count) {
-      /* Slowloris attack assumed */
-      goto attacked;
-    }
+  const size_t old_count = uuid_data(uuid).packet_count;
+  const size_t old_sent = uuid_data(uuid).sent;
 
-  } else {
-    flushed = uuid_data(uuid).rw_hooks->flush(uuid, uuid_data(uuid).rw_udata);
-    if (flushed < 0) {
-      goto test_errno;
-    }
-    if (flushed) {
-      goto flushed;
-    }
+  tmp = uuid_data(uuid).packet->write_func(fio_uuid2fd(uuid),
+                                           uuid_data(uuid).packet);
+  if (tmp <= 0) {
+    goto test_errno;
+  }
+
+  if (old_count >= 1024 && uuid_data(uuid).packet_count == old_count &&
+      uuid_data(uuid).sent >= old_sent &&
+      (uuid_data(uuid).sent - old_sent) < 32768) {
+    /* Slowloris attack assumed */
+    goto attacked;
   }
 
   /* end critical section */
@@ -2952,22 +2942,48 @@ ssize_t fio_flush(intptr_t uuid) {
 would_block:
   errno = EWOULDBLOCK;
   return -1;
+
 closed:
   fio_force_close(uuid);
   return -1;
+
+flush_rw_hook:
+  flushed = uuid_data(uuid).rw_hooks->flush(uuid, uuid_data(uuid).rw_udata);
+  fio_unlock(&uuid_data(uuid).sock_lock);
+  if (!flushed)
+    return 0;
+  if (flushed < 0) {
+    goto test_errno;
+  }
+  touchfd(fio_uuid2fd(uuid));
+  return 1;
+
 test_errno:
   fio_unlock(&uuid_data(uuid).sock_lock);
-  if (errno == EWOULDBLOCK || errno == EAGAIN || errno == ENOTCONN ||
-      errno == EINPROGRESS || errno == ENOSPC || errno == EINTR) {
+  switch (errno) {
+  case EWOULDBLOCK: /* ovreflow */
+#if EWOULDBLOCK != EAGAIN
+  case EAGAIN: /* ovreflow */
+#endif
+  case ENOTCONN:    /* ovreflow */
+  case EINPROGRESS: /* ovreflow */
+  case ENOSPC:      /* ovreflow */
+  case EINTR:
     return 1;
+  case EPIPE:  /* ovreflow */
+  case EIO:    /* ovreflow */
+  case EINVAL: /* ovreflow */
+  case EBADF:
+    uuid_data(uuid).close = 1;
+    fio_force_close(uuid);
+    return -1;
   }
   return 0;
 
+invalid:
+  /* bad UUID */
+  errno = EBADF;
   return -1;
-flushed:
-  touchfd(fio_uuid2fd(uuid));
-  fio_unlock(&uuid_data(uuid).sock_lock);
-  return 1;
 
 attacked:
   /* don't close, just detach from facil.io and mark uuid as invalid */
@@ -6508,6 +6524,8 @@ static inline void fio_publish2process2(int32_t filter, fio_str_info_s ch_name,
       fio_pubsub_create_message(filter, ch_name, msg, is_json, 1));
 }
 
+/* Sublime Text marker */
+void fio_publish___(fio_publish_args_s args);
 /**
  * Publishes a message to the relevant subscribers (if any).
  *
@@ -8763,6 +8781,8 @@ FIO_FUNC inline void fio_llist_test(void) {
     FIO_ASSERT_ALLOC(n);
     n->i = i;
     fio_ls_embd_push(&emlist, &n->node);
+    FIO_ASSERT(FIO_LS_EMBD_OBJ(struct fio_ls_test_s, node, emlist.next)->i == 0,
+               "fio_ls_embd_push should push to the end.");
   }
   FIO_ASSERT(fio_ls_embd_any(&emlist),
              "List should be populated after fio_ls_embd_push");
@@ -8794,6 +8814,8 @@ FIO_FUNC inline void fio_llist_test(void) {
     FIO_ASSERT_ALLOC(n)
     n->i = i;
     fio_ls_embd_unshift(&emlist, &n->node);
+    FIO_ASSERT(FIO_LS_EMBD_OBJ(struct fio_ls_test_s, node, emlist.next)->i == i,
+               "fio_ls_embd_unshift should push to the start.");
   }
   FIO_ASSERT(fio_ls_embd_any(&emlist),
              "List should be populated after fio_ls_embd_unshift");

data/ext/iodine/fio.h

@@ -427,9 +427,16 @@ int __attribute__((weak)) FIO_LOG_LEVEL;
       int len___log =                                                          \
           snprintf(tmp___log, FIO_LOG_LENGTH_LIMIT - 2, __VA_ARGS__);          \
       if (len___log <= 0 || len___log >= FIO_LOG_LENGTH_LIMIT - 2) {           \
-        fwrite("ERROR: log line output too long (can't write).", 46, 1,        \
-               stderr);                                                        \
-        break;                                                                 \
+        if (len___log >= (FIO_LOG_LENGTH_LIMIT >> 2) &&                        \
+            (FIO_LOG_LENGTH_LIMIT >> 2) + 52 < FIO_LOG_LENGTH_LIMIT) {         \
+          fwrite(tmp___log, (FIO_LOG_LENGTH_LIMIT >> 2), 1, stderr);           \
+          memcpy(tmp___log + (FIO_LOG_LENGTH_LIMIT >> 2),                      \
+                 "...\nERROR: log line output too long (can't write).\n", 52); \
+          len___log = (FIO_LOG_LENGTH_LIMIT >> 2) + 52;                        \
+        } else {                                                               \
+          fwrite("ERROR: log output error (can't write).\n", 39, 1, stderr);   \
+          break;                                                               \
+        }                                                                      \
       }                                                                        \
       tmp___log[len___log++] = '\n';                                           \
       tmp___log[len___log] = '0';                                              \

data/ext/iodine/http1.c

@@ -676,11 +676,8 @@ Connection Callbacks
 ***************************************************************************** */
 
 static inline void http1_consume_data(intptr_t uuid, http1pr_s *p) {
-  if (fio_pending(uuid) > 4) { /* throttle busy clients (slowloris) */
-    fio_suspend(uuid);
-    FIO_LOG_DEBUG("(HTTP/1,1) throttling client at %.*s",
-                  (int)fio_peer_addr(uuid).len, fio_peer_addr(uuid).data);
-    return;
+  if (fio_pending(uuid) > 4) {
+    goto throttle;
   }
   ssize_t i = 0;
   size_t org_len = p->buf_len;
@@ -720,6 +717,13 @@ static inline void http1_consume_data(intptr_t uuid, http1pr_s *p) {
   if (!pipeline_limit) {
     fio_force_event(uuid, FIO_EVENT_ON_DATA);
   }
+  return;
+
+throttle:
+  /* throttle busy clients (slowloris) */
+  fio_suspend(uuid);
+  FIO_LOG_DEBUG("(HTTP/1,1) throttling client at %.*s",
+                (int)fio_peer_addr(uuid).len, fio_peer_addr(uuid).data);
 }
 
 /** called when a data is available, but will not run concurrently */

data/ext/iodine/iodine_fiobj2rb.h

@@ -110,7 +110,7 @@ static inline VALUE fiobj2rb_deep(FIOBJ obj, uint8_t str2sym) {
   while (fiobj_ary_pop(data.stack))
     ;
   fiobj_free(data.stack);
-  IodineStore.remove(data.rb);
+  // IodineStore.remove(data.rb); // don't remove data
   return data.rb;
 }
 

data/ext/iodine/iodine_json.c

@@ -185,6 +185,7 @@ static inline VALUE iodine_json_convert(VALUE str, fiobj2rb_settings_s s) {
 //   }
 //   VALUE ret = fiobj2rb_deep(json, s.str2sym);
 //   fiobj_free(json);
+//   IodineStore.remove(ret);
 //   return ret;
 // }
 

data/ext/iodine/iodine_pubsub.c

@@ -181,6 +181,7 @@ static void iodine_pubsub_data_mark(void *c_) {
 }
 /* a callback for the GC (marking active objects) */
 static void iodine_pubsub_data_free(void *c_) {
+  FIO_LOG_DEBUG("iodine destroying engine");
   iodine_pubsub_s *data = c_;
   fio_pubsub_detach(data->engine);
   IodineStore.remove(data->handler); /* redundant except during exit */
@@ -411,20 +412,32 @@ static VALUE iodine_pubsub_redis_new(int argc, VALUE *argv, VALUE self) {
   (void)argv;
 }
 
+struct redis_callback_data {
+  FIOBJ response;
+  VALUE block;
+};
+
+/** A callback for Redis commands. */
+static void *iodine_pubsub_redis_callback_in_gil(void *data_) {
+  struct redis_callback_data *d = data_;
+  VALUE rb = Qnil;
+  if (!FIOBJ_IS_NULL(d->response)) {
+    rb = fiobj2rb_deep(d->response, 0);
+  }
+  IodineCaller.call2(d->block, call_id, 1, &rb);
+  IodineStore.remove(rb);
+  return NULL;
+}
+
 /** A callback for Redis commands. */
 static void iodine_pubsub_redis_callback(fio_pubsub_engine_s *e, FIOBJ response,
                                          void *udata) {
-  VALUE block = (VALUE)udata;
-  if (block == Qnil) {
+  struct redis_callback_data d = {.response = response, .block = (VALUE)udata};
+  if (d.block == Qnil) {
     return;
   }
-  VALUE rb = Qnil;
-  if (!FIOBJ_IS_NULL(response)) {
-    rb = IodineStore.add(fiobj2rb_deep(response, 0));
-  }
-  IodineCaller.call2(block, call_id, 1, &rb);
-  IodineStore.remove(rb);
-  IodineStore.remove(block);
+  IodineCaller.enterGVL(iodine_pubsub_redis_callback_in_gil, &d);
+  IodineStore.remove(d.block);
   (void)e;
 }
 

data/ext/iodine/redis_engine.c

@@ -44,8 +44,9 @@ typedef struct {
   size_t ref;
   fio_ls_embd_s queue;
   fio_lock_i lock;
+  fio_lock_i lock_connection;
   uint8_t ping_int;
-  uint8_t pub_send;
+  volatile uint8_t pub_sent;
   volatile uint8_t flag;
   uint8_t buf[];
 } redis_engine_s;
@@ -84,6 +85,7 @@ static inline void redis_internal_reset(struct redis_engine_internal_s *i) {
 static inline void redis_free(redis_engine_s *r) {
   if (fio_atomic_sub(&r->ref, 1))
     return;
+  FIO_LOG_DEBUG("freeing redis engine for %s:%s", r->address, r->port);
   redis_internal_reset(&r->pub_data);
   redis_internal_reset(&r->sub_data);
   fiobj_free(r->last_ch);
@@ -310,30 +312,42 @@ static void redis_perform_callback(void *e, void *cmd_) {
   fio_free(cmd);
 }
 
+/* send command within lock, to ensure flag integrity */
+static void redis_send_next_command_unsafe(redis_engine_s *r) {
+  if (!r->pub_sent && fio_ls_embd_any(&r->queue)) {
+    r->pub_sent = 1;
+    redis_commands_s *cmd =
+        FIO_LS_EMBD_OBJ(redis_commands_s, node, r->queue.next);
+    fio_write2(r->pub_data.uuid, .data.buffer = cmd->cmd,
+               .length = cmd->cmd_len, .after.dealloc = FIO_DEALLOC_NOOP);
+    FIO_LOG_DEBUG("(redis %d) Sending (%zu bytes):\n%s\n", getpid(),
+                  cmd->cmd_len, cmd->cmd);
+  }
+}
+
 /* attach a command to the queue */
 static void redis_attach_cmd(redis_engine_s *r, redis_commands_s *cmd) {
   fio_lock(&r->lock);
   fio_ls_embd_push(&r->queue, &cmd->node);
-  if (r->pub_send) {
-    fio_write2(r->pub_data.uuid, .data.buffer = cmd->cmd,
-               .length = cmd->cmd_len, .after.dealloc = FIO_DEALLOC_NOOP);
-    FIO_LOG_DEBUG("(%d) Sending (%zu bytes):\n%s\n", getpid(), cmd->cmd_len,
-                  cmd->cmd);
-  }
+  redis_send_next_command_unsafe(r);
   fio_unlock(&r->lock);
 }
 
 /** a local static callback, called when the RESP message is complete. */
 static void resp_on_pub_message(struct redis_engine_internal_s *i, FIOBJ msg) {
   redis_engine_s *r = pub2redis(i);
-#if DEBUG
-  FIOBJ json = fiobj_obj2json(msg, 1);
-  FIO_LOG_DEBUG("Redis reply:\n%s\n", fiobj_obj2cstr(json).data);
-  fiobj_free(json);
-#endif
+  // #if DEBUG
+  if (FIO_LOG_LEVEL >= FIO_LOG_LEVEL_DEBUG) {
+    FIOBJ json = fiobj_obj2json(msg, 1);
+    FIO_LOG_DEBUG("Redis reply:\n%s\n", fiobj_obj2cstr(json).data);
+    fiobj_free(json);
+  }
+  // #endif
   /* publishing / command parser */
   fio_lock(&r->lock);
   fio_ls_embd_s *node = fio_ls_embd_shift(&r->queue);
+  r->pub_sent = 0;
+  redis_send_next_command_unsafe(r);
   fio_unlock(&r->lock);
   if (!node) {
     /* TODO: possible ping? from server?! not likely... */
@@ -358,8 +372,8 @@ static void resp_on_sub_message(struct redis_engine_internal_s *i, FIOBJ msg) {
     if (FIOBJ_TYPE(msg) != FIOBJ_T_STRING || fiobj_obj2cstr(msg).len != 4 ||
         fiobj_obj2cstr(msg).data[0] != 'P') {
       FIO_LOG_WARNING("(redis) unexpected data format in "
-                      "subscription stream:\n     %s",
-                      fiobj_obj2cstr(msg).data);
+                      "subscription stream (%zu bytes):\n     %s\n",
+                      fiobj_obj2cstr(msg).len, fiobj_obj2cstr(msg).data);
     }
   } else {
     // FIOBJ *ary = fiobj_ary2ptr(msg);
@@ -410,6 +424,7 @@ static void redis_on_data(intptr_t uuid, fio_protocol_s *pr) {
                        REDIS_READ_BUFFER - internal->buf_pos);
   if (i <= 0)
     return;
+
   internal->buf_pos += i;
   i = resp_parse(&internal->parser, buf, internal->buf_pos);
   if (i) {
@@ -447,7 +462,7 @@ static void redis_on_close(intptr_t uuid, fio_protocol_s *pr) {
                       "Reconnecting...",
                       (int)getpid());
     }
-    r->pub_send = 0;
+    r->pub_sent = 0;
     fio_close(r->sub_data.uuid);
     redis_free(r);
   }
@@ -523,19 +538,16 @@ static void redis_on_connect(intptr_t uuid, void *i_) {
           (redis_commands_s){.cmd_len = r->auth_len, .callback = redis_on_auth};
       memcpy(cmd->cmd, r->auth, r->auth_len);
       fio_lock(&r->lock);
+      r->pub_sent = 0;
       fio_ls_embd_unshift(&r->queue, &cmd->node);
+      redis_send_next_command_unsafe(r);
+      fio_unlock(&r->lock);
+    } else {
+      fio_lock(&r->lock);
+      r->pub_sent = 0;
+      redis_send_next_command_unsafe(r);
       fio_unlock(&r->lock);
     }
-    fio_lock(&r->lock);
-    FIO_LS_EMBD_FOR(&r->queue, node) {
-      redis_commands_s *cmd = FIO_LS_EMBD_OBJ(redis_commands_s, node, node);
-      FIO_LOG_DEBUG("(%d) Sending (%zu bytes):\n%s\n", getpid(), cmd->cmd_len,
-                    cmd->cmd);
-      fio_write2(uuid, .data.buffer = cmd->cmd, .length = cmd->cmd_len,
-                 .after.dealloc = FIO_DEALLOC_NOOP);
-    }
-    r->pub_send = 1;
-    fio_unlock(&r->lock);
     FIO_LOG_INFO("(redis %d) publication connection established.",
                  (int)getpid());
   }
@@ -557,7 +569,9 @@ static void redis_on_connect_failed(intptr_t uuid, void *i_) {
 static void redis_connect(void *r_, void *i_) {
   redis_engine_s *r = r_;
   struct redis_engine_internal_s *i = i_;
+  fio_lock(&r->lock_connection);
   if (r->flag == 0 || i->uuid != -1 || !fio_is_running()) {
+    fio_unlock(&r->lock_connection);
     redis_free(r);
     return;
   }
@@ -565,6 +579,7 @@ static void redis_connect(void *r_, void *i_) {
   i->uuid = fio_connect(.address = r->address, .port = r->port,
                         .on_connect = redis_on_connect, .udata = i,
                         .on_fail = redis_on_connect_failed);
+  fio_unlock(&r->lock_connection);
 }
 
 /* *****************************************************************************
@@ -714,7 +729,7 @@ static void redis_forward_reply(fio_pubsub_engine_s *e, FIOBJ reply,
   }
   int32_t pid = (int32_t)fio_str2u32(data + 24);
   FIOBJ rp = fiobj_obj2json(reply, 0);
-  fio_publish(.filter = (-10 - pid), .channel.data = (char *)data,
+  fio_publish(.filter = (-10 - (int32_t)pid), .channel.data = (char *)data,
               .channel.len = 28, .message = fiobj_obj2cstr(rp), .is_json = 1);
   fiobj_free(rp);
 }
@@ -895,13 +910,16 @@ FIO_IGNORE_MACRO(struct redis_engine_create_args args) {
       .cmd_reply =
           fio_subscribe(.filter = -10 - (uint32_t)getpid(), .udata1 = r,
                         .on_message = redis_on_internal_reply),
-      .address = ((char *)(r + 1) + 0),
-      .port = ((char *)(r + 1) + args.address.len + 1),
-      .auth = ((char *)(r + 1) + args.address.len + args.port.len + 2),
+      .address = ((char *)(r + 1) + (REDIS_READ_BUFFER * 2)),
+      .port =
+          ((char *)(r + 1) + (REDIS_READ_BUFFER * 2) + args.address.len + 1),
+      .auth = ((char *)(r + 1) + (REDIS_READ_BUFFER * 2) + args.address.len +
+               args.port.len + 2),
       .auth_len = args.auth.len,
       .ref = 1,
       .queue = FIO_LS_INIT(r->queue),
       .lock = FIO_LOCK_INIT,
+      .lock_connection = FIO_LOCK_INIT,
       .ping_int = args.ping_interval,
       .flag = 1,
   };

data/lib/iodine/version.rb

@@ -1,3 +1,3 @@
 module Iodine
-  VERSION = '0.7.20'.freeze
+  VERSION = '0.7.21'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: iodine
 version: !ruby/object:Gem::Version
-  version: 0.7.20
+  version: 0.7.21
 platform: ruby
 authors:
 - Boaz Segev
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-01-31 00:00:00.000000000 Z
+date: 2019-02-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -220,7 +220,7 @@ licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-post_install_message: 'Thank you for installing Iodine 0.7.20.
+post_install_message: 'Thank you for installing Iodine 0.7.21.
 
 '
 rdoc_options: []