nutcracker 0.4.0.16 → 0.4.1.18

data/ext/nutcracker/src/nc_core.c

@@ -311,7 +311,14 @@ core_core(void *arg, uint32_t events)
 {
     rstatus_t status;
     struct conn *conn = arg;
-    struct context *ctx = conn_to_ctx(conn);
+    struct context *ctx;
+
+    if (conn->owner == NULL) {
+        log_warn("conn is already unrefed!");
+        return NC_OK;
+    }
+
+    ctx = conn_to_ctx(conn);
 
     log_debug(LOG_VVERB, "event %04"PRIX32" on %c %d", events,
               conn->client ? 'c' : (conn->proxy ? 'p' : 's'), conn->sd);

data/ext/nutcracker/src/nc_message.c

@@ -226,6 +226,7 @@ done:
     msg->token = NULL;
 
     msg->parser = NULL;
+    msg->add_auth = NULL;
     msg->result = MSG_PARSE_OK;
 
     msg->fragment = NULL;
@@ -292,8 +293,10 @@ msg_get(struct conn *conn, bool request, bool redis)
         } else {
             msg->parser = redis_parse_rsp;
         }
+        msg->add_auth = redis_add_auth;
         msg->fragment = redis_fragment;
         msg->reply = redis_reply;
+        msg->failure = redis_failure;
         msg->pre_coalesce = redis_pre_coalesce;
         msg->post_coalesce = redis_post_coalesce;
     } else {
@@ -302,7 +305,9 @@ msg_get(struct conn *conn, bool request, bool redis)
         } else {
             msg->parser = memcache_parse_rsp;
         }
+        msg->add_auth = memcache_add_auth;
         msg->fragment = memcache_fragment;
+        msg->failure = memcache_failure;
         msg->pre_coalesce = memcache_pre_coalesce;
         msg->post_coalesce = memcache_post_coalesce;
     }
@@ -472,11 +477,13 @@ msg_ensure_mbuf(struct msg *msg, size_t len)
     } else {
         mbuf = STAILQ_LAST(&msg->mhdr, mbuf, next);
     }
+
     return mbuf;
 }
 
 /*
- * append small(small than a mbuf) content into msg
+ * Append n bytes of data, with n <= mbuf_size(mbuf)
+ * into mbuf
  */
 rstatus_t
 msg_append(struct msg *msg, uint8_t *pos, size_t n)
@@ -494,11 +501,13 @@ msg_append(struct msg *msg, uint8_t *pos, size_t n)
 
     mbuf_copy(mbuf, pos, n);
     msg->mlen += (uint32_t)n;
+
     return NC_OK;
 }
 
 /*
- * prepend small(small than a mbuf) content into msg
+ * Prepend n bytes of data, with n <= mbuf_size(mbuf)
+ * into mbuf
  */
 rstatus_t
 msg_prepend(struct msg *msg, uint8_t *pos, size_t n)
@@ -516,17 +525,20 @@ msg_prepend(struct msg *msg, uint8_t *pos, size_t n)
     msg->mlen += (uint32_t)n;
 
     STAILQ_INSERT_HEAD(&msg->mhdr, mbuf, next);
+
     return NC_OK;
 }
 
 /*
- * prepend small(small than a mbuf) content into msg
+ * Prepend a formatted string into msg. Returns an error if the formatted
+ * string does not fit in a single mbuf.
  */
 rstatus_t
 msg_prepend_format(struct msg *msg, const char *fmt, ...)
 {
     struct mbuf *mbuf;
-    int32_t n;
+    int n;
+    uint32_t size;
     va_list args;
 
     mbuf = mbuf_get();
@@ -534,15 +546,19 @@ msg_prepend_format(struct msg *msg, const char *fmt, ...)
         return NC_ENOMEM;
     }
 
+    size = mbuf_size(mbuf);
+
     va_start(args, fmt);
-    n = nc_vscnprintf(mbuf->last, mbuf_size(mbuf), fmt, args);
+    n = nc_vsnprintf(mbuf->last, size, fmt, args);
     va_end(args);
+    if (n <= 0 || n >= (int)size) {
+        return NC_ERROR;
+    }
 
     mbuf->last += n;
     msg->mlen += (uint32_t)n;
-
-    ASSERT(mbuf_size(mbuf) >= 0);
     STAILQ_INSERT_HEAD(&msg->mhdr, mbuf, next);
+
     return NC_OK;
 }
 

data/ext/nutcracker/src/nc_message.h

@@ -21,9 +21,11 @@
 #include <nc_core.h>
 
 typedef void (*msg_parse_t)(struct msg *);
+typedef rstatus_t (*msg_add_auth_t)(struct context *ctx, struct conn *c_conn, struct conn *s_conn);
 typedef rstatus_t (*msg_fragment_t)(struct msg *, uint32_t, struct msg_tqh *);
 typedef void (*msg_coalesce_t)(struct msg *r);
 typedef rstatus_t (*msg_reply_t)(struct msg *r);
+typedef bool (*msg_failure_t)(struct msg *r);
 
 typedef enum msg_parse_result {
     MSG_PARSE_OK,                         /* parsing ok */
@@ -45,6 +47,7 @@ typedef enum msg_parse_result {
     ACTION( REQ_MC_PREPEND )                                                                        \
     ACTION( REQ_MC_INCR )                      /* memcache arithmetic request */                    \
     ACTION( REQ_MC_DECR )                                                                           \
+    ACTION( REQ_MC_TOUCH )                     /* memcache touch request */                         \
     ACTION( REQ_MC_QUIT )                      /* memcache quit request */                          \
     ACTION( RSP_MC_NUM )                       /* memcache arithmetic response */                   \
     ACTION( RSP_MC_STORED )                    /* memcache cas and storage response */              \
@@ -54,6 +57,7 @@ typedef enum msg_parse_result {
     ACTION( RSP_MC_END )                                                                            \
     ACTION( RSP_MC_VALUE )                                                                          \
     ACTION( RSP_MC_DELETED )                   /* memcache delete response */                       \
+    ACTION( RSP_MC_TOUCHED )                   /* memcache touch response */                        \
     ACTION( RSP_MC_ERROR )                     /* memcache error responses */                       \
     ACTION( RSP_MC_CLIENT_ERROR )                                                                   \
     ACTION( RSP_MC_SERVER_ERROR )                                                                   \
@@ -160,8 +164,23 @@ typedef enum msg_parse_result {
     ACTION( REQ_REDIS_EVALSHA )                                                                     \
     ACTION( REQ_REDIS_PING )                   /* redis requests - ping/quit */                     \
     ACTION( REQ_REDIS_QUIT)                                                                         \
+    ACTION( REQ_REDIS_AUTH)                                                                         \
+    ACTION( REQ_REDIS_SELECT)                  /* only during init */                               \
     ACTION( RSP_REDIS_STATUS )                 /* redis response */                                 \
     ACTION( RSP_REDIS_ERROR )                                                                       \
+    ACTION( RSP_REDIS_ERROR_ERR )                                                                   \
+    ACTION( RSP_REDIS_ERROR_OOM )                                                                   \
+    ACTION( RSP_REDIS_ERROR_BUSY )                                                                  \
+    ACTION( RSP_REDIS_ERROR_NOAUTH )                                                                \
+    ACTION( RSP_REDIS_ERROR_LOADING )                                                               \
+    ACTION( RSP_REDIS_ERROR_BUSYKEY )                                                               \
+    ACTION( RSP_REDIS_ERROR_MISCONF )                                                               \
+    ACTION( RSP_REDIS_ERROR_NOSCRIPT )                                                              \
+    ACTION( RSP_REDIS_ERROR_READONLY )                                                              \
+    ACTION( RSP_REDIS_ERROR_WRONGTYPE )                                                             \
+    ACTION( RSP_REDIS_ERROR_EXECABORT )                                                             \
+    ACTION( RSP_REDIS_ERROR_MASTERDOWN )                                                            \
+    ACTION( RSP_REDIS_ERROR_NOREPLICAS )                                                            \
     ACTION( RSP_REDIS_INTEGER )                                                                     \
     ACTION( RSP_REDIS_BULK )                                                                        \
     ACTION( RSP_REDIS_MULTIBULK )                                                                   \
@@ -202,7 +221,10 @@ struct msg {
     msg_parse_result_t   result;          /* message parsing result */
 
     msg_fragment_t       fragment;        /* message fragment */
-    msg_reply_t          reply;           /* gen message reply (example: ping) */
+    msg_reply_t          reply;           /* generate message reply (example: ping) */
+    msg_add_auth_t       add_auth;        /* add auth message when we forward msg */
+    msg_failure_t        failure;         /* transient failure response? */
+
     msg_coalesce_t       pre_coalesce;    /* message pre-coalesce */
     msg_coalesce_t       post_coalesce;   /* message post-coalesce */
 
@@ -267,6 +289,7 @@ void req_put(struct msg *msg);
 bool req_done(struct conn *conn, struct msg *msg);
 bool req_error(struct conn *conn, struct msg *msg);
 void req_server_enqueue_imsgq(struct context *ctx, struct conn *conn, struct msg *msg);
+void req_server_enqueue_imsgq_head(struct context *ctx, struct conn *conn, struct msg *msg);
 void req_server_dequeue_imsgq(struct context *ctx, struct conn *conn, struct msg *msg);
 void req_client_enqueue_omsgq(struct context *ctx, struct conn *conn, struct msg *msg);
 void req_server_enqueue_omsgq(struct context *ctx, struct conn *conn, struct msg *msg);

data/ext/nutcracker/src/nc_proxy.c

@@ -15,6 +15,7 @@
  * limitations under the License.
  */
 
+#include <sys/stat.h>
 #include <sys/un.h>
 
 #include <nc_core.h>
@@ -29,9 +30,9 @@ proxy_ref(struct conn *conn, void *owner)
     ASSERT(!conn->client && conn->proxy);
     ASSERT(conn->owner == NULL);
 
-    conn->family = pool->family;
-    conn->addrlen = pool->addrlen;
-    conn->addr = pool->addr;
+    conn->family = pool->info.family;
+    conn->addrlen = pool->info.addrlen;
+    conn->addr = (struct sockaddr *)&pool->info.addr;
 
     pool->p_conn = conn;
 
@@ -148,6 +149,16 @@ proxy_listen(struct context *ctx, struct conn *p)
         return NC_ERROR;
     }
 
+    if (p->family == AF_UNIX && pool->perm) {
+        struct sockaddr_un *un = (struct sockaddr_un *)p->addr;
+        status = chmod(un->sun_path, pool->perm);
+        if (status < 0) {
+            log_error("chmod on p %d on addr '%.*s' failed: %s", p->sd,
+                      pool->addrstr.len, pool->addrstr.data, strerror(errno));
+            return NC_ERROR;
+        }
+    }
+
     status = listen(p->sd, pool->backlog);
     if (status < 0) {
         log_error("listen on p %d on addr '%.*s' failed: %s", p->sd,

data/ext/nutcracker/src/nc_rbtree.c

@@ -218,11 +218,7 @@ rbtree_delete(struct rbtree *tree, struct rbnode *node)
         subst = node;
     } else {
         subst = rbtree_node_min(node->right, sentinel);
-        if (subst->left != sentinel) {
-            temp = subst->left;
-        } else {
-            temp = subst->right;
-        }
+        temp = subst->right;
     }
 
     if (subst == *root) {

data/ext/nutcracker/src/nc_request.c

@@ -303,7 +303,7 @@ req_server_enqueue_imsgq(struct context *ctx, struct conn *conn, struct msg *msg
      * or the message is dequeued from the server out_q
      *
      * noreply request are free from timeouts because client is not intrested
-     * in the reponse anyway!
+     * in the response anyway!
      */
     if (!msg->noreply) {
         msg_tmo_insert(msg, conn);
@@ -315,6 +315,30 @@ req_server_enqueue_imsgq(struct context *ctx, struct conn *conn, struct msg *msg
     stats_server_incr_by(ctx, conn->owner, in_queue_bytes, msg->mlen);
 }
 
+void
+req_server_enqueue_imsgq_head(struct context *ctx, struct conn *conn, struct msg *msg)
+{
+    ASSERT(msg->request);
+    ASSERT(!conn->client && !conn->proxy);
+
+    /*
+     * timeout clock starts ticking the instant the message is enqueued into
+     * the server in_q; the clock continues to tick until it either expires
+     * or the message is dequeued from the server out_q
+     *
+     * noreply request are free from timeouts because client is not intrested
+     * in the reponse anyway!
+     */
+    if (!msg->noreply) {
+        msg_tmo_insert(msg, conn);
+    }
+
+    TAILQ_INSERT_HEAD(&conn->imsg_q, msg, s_tqe);
+
+    stats_server_incr(ctx, conn->owner, in_queue);
+    stats_server_incr_by(ctx, conn->owner, in_queue_bytes, msg->mlen);
+}
+
 void
 req_server_dequeue_imsgq(struct context *ctx, struct conn *conn, struct msg *msg)
 {
@@ -429,20 +453,21 @@ req_recv_next(struct context *ctx, struct conn *conn, bool alloc)
 static rstatus_t
 req_make_reply(struct context *ctx, struct conn *conn, struct msg *req)
 {
-    struct msg *msg;
+    struct msg *rsp;
 
-    msg = msg_get(conn, true, conn->redis); /* replay */
-    if (msg == NULL) {
+    rsp = msg_get(conn, false, conn->redis); /* replay */
+    if (rsp == NULL) {
         conn->err = errno;
         return NC_ENOMEM;
     }
 
-    req->peer = msg;
-    msg->peer = req;
-    msg->request = 0;
+    req->peer = rsp;
+    rsp->peer = req;
+    rsp->request = 0;
 
     req->done = 1;
     conn->enqueue_outq(ctx, conn, req);
+
     return NC_OK;
 }
 
@@ -460,19 +485,32 @@ req_filter(struct context *ctx, struct conn *conn, struct msg *msg)
     }
 
     /*
-     * Handle "quit\r\n", which is the protocol way of doing a
-     * passive close
+     * Handle "quit\r\n" (memcache) or "*1\r\n$4\r\nquit\r\n" (redis), which
+     * is the protocol way of doing a passive close. The connection is closed
+     * as soon as all pending replies have been written to the client.
      */
     if (msg->quit) {
-        ASSERT(conn->rmsg == NULL);
         log_debug(LOG_INFO, "filter quit req %"PRIu64" from c %d", msg->id,
                   conn->sd);
+        if (conn->rmsg != NULL) {
+            log_debug(LOG_INFO, "discard invalid req %"PRIu64" len %"PRIu32" "
+                      "from c %d sent after quit req", conn->rmsg->id,
+                      conn->rmsg->mlen, conn->sd);
+        }
         conn->eof = 1;
         conn->recv_ready = 0;
         req_put(msg);
         return true;
     }
 
+    /*
+     * If this conn is not authenticated, we will mark it as noforward,
+     * and handle it in the redis_reply handler.
+     */
+    if (!conn_authenticated(conn)) {
+        msg->noforward = 1;
+    }
+
     return false;
 }
 
@@ -554,6 +592,16 @@ req_forward(struct context *ctx, struct conn *c_conn, struct msg *msg)
             return;
         }
     }
+
+    if (!conn_authenticated(s_conn)) {
+        status = msg->add_auth(ctx, c_conn, s_conn);
+        if (status != NC_OK) {
+            req_forward_error(ctx, c_conn, msg);
+            s_conn->err = errno;
+            return;
+        }
+    }
+
     s_conn->enqueue_inq(ctx, s_conn, msg);
 
     req_forward_stats(ctx, s_conn->owner, msg);

data/ext/nutcracker/src/nc_response.c

@@ -171,7 +171,7 @@ rsp_filter(struct context *ctx, struct conn *conn, struct msg *msg)
          * We handle this stray packet scenario in nutcracker by closing the
          * server connection which would end up sending SERVER_ERROR to all
          * clients that have requests pending on this server connection. The
-         * fix is aggresive, but not doing so would lead to clients getting
+         * fix is aggressive, but not doing so would lead to clients getting
          * out of sync with the server and as a result clients end up getting
          * responses that don't correspond to the right request.
          *
@@ -184,7 +184,28 @@ rsp_filter(struct context *ctx, struct conn *conn, struct msg *msg)
     ASSERT(pmsg->peer == NULL);
     ASSERT(pmsg->request && !pmsg->done);
 
+    /*
+     * If the response from a server suggests a protocol level transient
+     * failure, close the server connection and send back a generic error
+     * response to the client.
+     *
+     * If auto_eject_host is enabled, this will also update the failure_count
+     * and eject the server if it exceeds the failure_limit
+     */
+    if (msg->failure(msg)) {
+        log_debug(LOG_INFO, "server failure rsp %"PRIu64" len %"PRIu32" "
+                  "type %d on s %d", msg->id, msg->mlen, msg->type, conn->sd);
+        rsp_put(msg);
+
+        conn->err = EINVAL;
+        conn->done = 1;
+
+        return true;
+    }
+
     if (pmsg->swallow) {
+        conn->swallow_msg(conn, pmsg, msg);
+
         conn->dequeue_outq(ctx, conn, pmsg);
         pmsg->done = 1;
 
@@ -201,12 +222,12 @@ rsp_filter(struct context *ctx, struct conn *conn, struct msg *msg)
 }
 
 static void
-rsp_forward_stats(struct context *ctx, struct server *server, struct msg *msg)
+rsp_forward_stats(struct context *ctx, struct server *server, struct msg *msg, uint32_t msgsize)
 {
     ASSERT(!msg->request);
 
     stats_server_incr(ctx, server, responses);
-    stats_server_incr_by(ctx, server, response_bytes, msg->mlen);
+    stats_server_incr_by(ctx, server, response_bytes, msgsize);
 }
 
 static void
@@ -215,8 +236,10 @@ rsp_forward(struct context *ctx, struct conn *s_conn, struct msg *msg)
     rstatus_t status;
     struct msg *pmsg;
     struct conn *c_conn;
+    uint32_t msgsize;
 
     ASSERT(!s_conn->client && !s_conn->proxy);
+    msgsize = msg->mlen;
 
     /* response from server implies that server is ok and heartbeating */
     server_ok(ctx, s_conn);
@@ -245,7 +268,7 @@ rsp_forward(struct context *ctx, struct conn *s_conn, struct msg *msg)
         }
     }
 
-    rsp_forward_stats(ctx, s_conn->owner, msg);
+    rsp_forward_stats(ctx, s_conn->owner, msg, msgsize);
 }
 
 void

data/ext/nutcracker/src/nc_server.c

@@ -22,6 +22,23 @@
 #include <nc_server.h>
 #include <nc_conf.h>
 
+static void
+server_resolve(struct server *server, struct conn *conn)
+{
+    rstatus_t status;
+
+    status = nc_resolve(&server->addrstr, server->port, &server->info);
+    if (status != NC_OK) {
+        conn->err = EHOSTDOWN;
+        conn->done = 1;
+        return;
+    }
+
+    conn->family = server->info.family;
+    conn->addrlen = server->info.addrlen;
+    conn->addr = (struct sockaddr *)&server->info.addr;
+}
+
 void
 server_ref(struct conn *conn, void *owner)
 {
@@ -30,9 +47,7 @@ server_ref(struct conn *conn, void *owner)
     ASSERT(!conn->client && !conn->proxy);
     ASSERT(conn->owner == NULL);
 
-    conn->family = server->family;
-    conn->addrlen = server->addrlen;
-    conn->addr = server->addr;
+    server_resolve(server, conn);
 
     server->ns_conn_q++;
     TAILQ_INSERT_TAIL(&server->s_conn_q, conn, conn_tqe);
@@ -338,6 +353,8 @@ server_close(struct context *ctx, struct conn *conn)
     server_close_stats(ctx, conn->owner, conn->err, conn->eof,
                        conn->connected);
 
+    conn->connected = false;
+
     if (conn->sd < 0) {
         server_failure(ctx, conn->owner);
         conn->unref(conn);
@@ -452,6 +469,12 @@ server_connect(struct context *ctx, struct server *server, struct conn *conn)
 
     ASSERT(!conn->client && !conn->proxy);
 
+    if (conn->err) {
+      ASSERT(conn->done && conn->sd < 0);
+      errno = conn->err;
+      return NC_ERROR;
+    }
+
     if (conn->sd > 0) {
         /* already connected on server connection */
         return NC_OK;
@@ -535,6 +558,8 @@ server_connected(struct context *ctx, struct conn *conn)
     conn->connecting = 0;
     conn->connected = 1;
 
+    conn->post_connect(ctx, conn, server);
+
     log_debug(LOG_INFO, "connected on s %d to server '%.*s'", conn->sd,
               server->pname.len, server->pname.data);
 }
@@ -605,12 +630,15 @@ static uint32_t
 server_pool_hash(struct server_pool *pool, uint8_t *key, uint32_t keylen)
 {
     ASSERT(array_n(&pool->server) != 0);
+    ASSERT(key != NULL);
 
     if (array_n(&pool->server) == 1) {
         return 0;
     }
 
-    ASSERT(key != NULL && keylen != 0);
+    if (keylen == 0) {
+        return 0;
+    }
 
     return pool->key_hash((char *)key, keylen);
 }
@@ -621,7 +649,7 @@ server_pool_idx(struct server_pool *pool, uint8_t *key, uint32_t keylen)
     uint32_t hash, idx;
 
     ASSERT(array_n(&pool->server) != 0);
-    ASSERT(key != NULL && keylen != 0);
+    ASSERT(key != NULL);
 
     /*
      * If hash_tag: is configured for this server pool, we use the part of