nutcracker 0.3.0.12 → 0.4.0.13

data/ext/nutcracker/src/nc_message.h

@@ -21,139 +21,163 @@
 #include <nc_core.h>
 
 typedef void (*msg_parse_t)(struct msg *);
-typedef rstatus_t (*msg_post_splitcopy_t)(struct msg *);
+typedef rstatus_t (*msg_fragment_t)(struct msg *, uint32_t, struct msg_tqh *);
 typedef void (*msg_coalesce_t)(struct msg *r);
+typedef rstatus_t (*msg_reply_t)(struct msg *r);
 
 typedef enum msg_parse_result {
     MSG_PARSE_OK,                         /* parsing ok */
     MSG_PARSE_ERROR,                      /* parsing error */
     MSG_PARSE_REPAIR,                     /* more to parse -> repair parsed & unparsed data */
-    MSG_PARSE_FRAGMENT,                   /* multi-vector request -> fragment */
     MSG_PARSE_AGAIN,                      /* incomplete -> parse again */
 } msg_parse_result_t;
 
+#define MSG_TYPE_CODEC(ACTION)                                                                      \
+    ACTION( UNKNOWN )                                                                               \
+    ACTION( REQ_MC_GET )                       /* memcache retrieval requests */                    \
+    ACTION( REQ_MC_GETS )                                                                           \
+    ACTION( REQ_MC_DELETE )                    /* memcache delete request */                        \
+    ACTION( REQ_MC_CAS )                       /* memcache cas request and storage request */       \
+    ACTION( REQ_MC_SET )                       /* memcache storage request */                       \
+    ACTION( REQ_MC_ADD )                                                                            \
+    ACTION( REQ_MC_REPLACE )                                                                        \
+    ACTION( REQ_MC_APPEND )                                                                         \
+    ACTION( REQ_MC_PREPEND )                                                                        \
+    ACTION( REQ_MC_INCR )                      /* memcache arithmetic request */                    \
+    ACTION( REQ_MC_DECR )                                                                           \
+    ACTION( REQ_MC_QUIT )                      /* memcache quit request */                          \
+    ACTION( RSP_MC_NUM )                       /* memcache arithmetic response */                   \
+    ACTION( RSP_MC_STORED )                    /* memcache cas and storage response */              \
+    ACTION( RSP_MC_NOT_STORED )                                                                     \
+    ACTION( RSP_MC_EXISTS )                                                                         \
+    ACTION( RSP_MC_NOT_FOUND )                                                                      \
+    ACTION( RSP_MC_END )                                                                            \
+    ACTION( RSP_MC_VALUE )                                                                          \
+    ACTION( RSP_MC_DELETED )                   /* memcache delete response */                       \
+    ACTION( RSP_MC_ERROR )                     /* memcache error responses */                       \
+    ACTION( RSP_MC_CLIENT_ERROR )                                                                   \
+    ACTION( RSP_MC_SERVER_ERROR )                                                                   \
+    ACTION( REQ_REDIS_DEL )                    /* redis commands - keys */                          \
+    ACTION( REQ_REDIS_EXISTS )                                                                      \
+    ACTION( REQ_REDIS_EXPIRE )                                                                      \
+    ACTION( REQ_REDIS_EXPIREAT )                                                                    \
+    ACTION( REQ_REDIS_PEXPIRE )                                                                     \
+    ACTION( REQ_REDIS_PEXPIREAT )                                                                   \
+    ACTION( REQ_REDIS_PERSIST )                                                                     \
+    ACTION( REQ_REDIS_PTTL )                                                                        \
+    ACTION( REQ_REDIS_SORT )                                                                        \
+    ACTION( REQ_REDIS_TTL )                                                                         \
+    ACTION( REQ_REDIS_TYPE )                                                                        \
+    ACTION( REQ_REDIS_APPEND )                 /* redis requests - string */                        \
+    ACTION( REQ_REDIS_BITCOUNT )                                                                    \
+    ACTION( REQ_REDIS_DECR )                                                                        \
+    ACTION( REQ_REDIS_DECRBY )                                                                      \
+    ACTION( REQ_REDIS_DUMP )                                                                        \
+    ACTION( REQ_REDIS_GET )                                                                         \
+    ACTION( REQ_REDIS_GETBIT )                                                                      \
+    ACTION( REQ_REDIS_GETRANGE )                                                                    \
+    ACTION( REQ_REDIS_GETSET )                                                                      \
+    ACTION( REQ_REDIS_INCR )                                                                        \
+    ACTION( REQ_REDIS_INCRBY )                                                                      \
+    ACTION( REQ_REDIS_INCRBYFLOAT )                                                                 \
+    ACTION( REQ_REDIS_MGET )                                                                        \
+    ACTION( REQ_REDIS_MSET )                                                                        \
+    ACTION( REQ_REDIS_PSETEX )                                                                      \
+    ACTION( REQ_REDIS_RESTORE )                                                                     \
+    ACTION( REQ_REDIS_SET )                                                                         \
+    ACTION( REQ_REDIS_SETBIT )                                                                      \
+    ACTION( REQ_REDIS_SETEX )                                                                       \
+    ACTION( REQ_REDIS_SETNX )                                                                       \
+    ACTION( REQ_REDIS_SETRANGE )                                                                    \
+    ACTION( REQ_REDIS_STRLEN )                                                                      \
+    ACTION( REQ_REDIS_HDEL )                   /* redis requests - hashes */                        \
+    ACTION( REQ_REDIS_HEXISTS )                                                                     \
+    ACTION( REQ_REDIS_HGET )                                                                        \
+    ACTION( REQ_REDIS_HGETALL )                                                                     \
+    ACTION( REQ_REDIS_HINCRBY )                                                                     \
+    ACTION( REQ_REDIS_HINCRBYFLOAT )                                                                \
+    ACTION( REQ_REDIS_HKEYS )                                                                       \
+    ACTION( REQ_REDIS_HLEN )                                                                        \
+    ACTION( REQ_REDIS_HMGET )                                                                       \
+    ACTION( REQ_REDIS_HMSET )                                                                       \
+    ACTION( REQ_REDIS_HSET )                                                                        \
+    ACTION( REQ_REDIS_HSETNX )                                                                      \
+    ACTION( REQ_REDIS_HSCAN)                                                                        \
+    ACTION( REQ_REDIS_HVALS )                                                                       \
+    ACTION( REQ_REDIS_LINDEX )                 /* redis requests - lists */                         \
+    ACTION( REQ_REDIS_LINSERT )                                                                     \
+    ACTION( REQ_REDIS_LLEN )                                                                        \
+    ACTION( REQ_REDIS_LPOP )                                                                        \
+    ACTION( REQ_REDIS_LPUSH )                                                                       \
+    ACTION( REQ_REDIS_LPUSHX )                                                                      \
+    ACTION( REQ_REDIS_LRANGE )                                                                      \
+    ACTION( REQ_REDIS_LREM )                                                                        \
+    ACTION( REQ_REDIS_LSET )                                                                        \
+    ACTION( REQ_REDIS_LTRIM )                                                                       \
+    ACTION( REQ_REDIS_PFADD )                  /* redis requests - hyperloglog */                   \
+    ACTION( REQ_REDIS_PFCOUNT )                                                                     \
+    ACTION( REQ_REDIS_PFMERGE )                                                                     \
+    ACTION( REQ_REDIS_RPOP )                                                                        \
+    ACTION( REQ_REDIS_RPOPLPUSH )                                                                   \
+    ACTION( REQ_REDIS_RPUSH )                                                                       \
+    ACTION( REQ_REDIS_RPUSHX )                                                                      \
+    ACTION( REQ_REDIS_SADD )                   /* redis requests - sets */                          \
+    ACTION( REQ_REDIS_SCARD )                                                                       \
+    ACTION( REQ_REDIS_SDIFF )                                                                       \
+    ACTION( REQ_REDIS_SDIFFSTORE )                                                                  \
+    ACTION( REQ_REDIS_SINTER )                                                                      \
+    ACTION( REQ_REDIS_SINTERSTORE )                                                                 \
+    ACTION( REQ_REDIS_SISMEMBER )                                                                   \
+    ACTION( REQ_REDIS_SMEMBERS )                                                                    \
+    ACTION( REQ_REDIS_SMOVE )                                                                       \
+    ACTION( REQ_REDIS_SPOP )                                                                        \
+    ACTION( REQ_REDIS_SRANDMEMBER )                                                                 \
+    ACTION( REQ_REDIS_SREM )                                                                        \
+    ACTION( REQ_REDIS_SUNION )                                                                      \
+    ACTION( REQ_REDIS_SUNIONSTORE )                                                                 \
+    ACTION( REQ_REDIS_SSCAN)                                                                        \
+    ACTION( REQ_REDIS_ZADD )                   /* redis requests - sorted sets */                   \
+    ACTION( REQ_REDIS_ZCARD )                                                                       \
+    ACTION( REQ_REDIS_ZCOUNT )                                                                      \
+    ACTION( REQ_REDIS_ZINCRBY )                                                                     \
+    ACTION( REQ_REDIS_ZINTERSTORE )                                                                 \
+    ACTION( REQ_REDIS_ZLEXCOUNT )                                                                   \
+    ACTION( REQ_REDIS_ZRANGE )                                                                      \
+    ACTION( REQ_REDIS_ZRANGEBYLEX )                                                                 \
+    ACTION( REQ_REDIS_ZRANGEBYSCORE )                                                               \
+    ACTION( REQ_REDIS_ZRANK )                                                                       \
+    ACTION( REQ_REDIS_ZREM )                                                                        \
+    ACTION( REQ_REDIS_ZREMRANGEBYRANK )                                                             \
+    ACTION( REQ_REDIS_ZREMRANGEBYLEX )                                                              \
+    ACTION( REQ_REDIS_ZREMRANGEBYSCORE )                                                            \
+    ACTION( REQ_REDIS_ZREVRANGE )                                                                   \
+    ACTION( REQ_REDIS_ZREVRANGEBYSCORE )                                                            \
+    ACTION( REQ_REDIS_ZREVRANK )                                                                    \
+    ACTION( REQ_REDIS_ZSCORE )                                                                      \
+    ACTION( REQ_REDIS_ZUNIONSTORE )                                                                 \
+    ACTION( REQ_REDIS_ZSCAN)                                                                        \
+    ACTION( REQ_REDIS_EVAL )                   /* redis requests - eval */                          \
+    ACTION( REQ_REDIS_EVALSHA )                                                                     \
+    ACTION( REQ_REDIS_PING )                   /* redis requests - ping/quit */                     \
+    ACTION( REQ_REDIS_QUIT)                                                                         \
+    ACTION( RSP_REDIS_STATUS )                 /* redis response */                                 \
+    ACTION( RSP_REDIS_ERROR )                                                                       \
+    ACTION( RSP_REDIS_INTEGER )                                                                     \
+    ACTION( RSP_REDIS_BULK )                                                                        \
+    ACTION( RSP_REDIS_MULTIBULK )                                                                   \
+    ACTION( SENTINEL )                                                                              \
+
+
+#define DEFINE_ACTION(_name) MSG_##_name,
 typedef enum msg_type {
-    MSG_UNKNOWN,
-    MSG_REQ_MC_GET,                       /* memcache retrieval requests */
-    MSG_REQ_MC_GETS,
-    MSG_REQ_MC_DELETE,                    /* memcache delete request */
-    MSG_REQ_MC_CAS,                       /* memcache cas request and storage request */
-    MSG_REQ_MC_SET,                       /* memcache storage request */
-    MSG_REQ_MC_ADD,
-    MSG_REQ_MC_REPLACE,
-    MSG_REQ_MC_APPEND,
-    MSG_REQ_MC_PREPEND,
-    MSG_REQ_MC_INCR,                      /* memcache arithmetic request */
-    MSG_REQ_MC_DECR,
-    MSG_REQ_MC_QUIT,                      /* memcache quit request */
-    MSG_RSP_MC_NUM,                       /* memcache arithmetic response */
-    MSG_RSP_MC_STORED,                    /* memcache cas and storage response */
-    MSG_RSP_MC_NOT_STORED,
-    MSG_RSP_MC_EXISTS,
-    MSG_RSP_MC_NOT_FOUND,
-    MSG_RSP_MC_END,
-    MSG_RSP_MC_VALUE,
-    MSG_RSP_MC_DELETED,                   /* memcache delete response */
-    MSG_RSP_MC_ERROR,                     /* memcache error responses */
-    MSG_RSP_MC_CLIENT_ERROR,
-    MSG_RSP_MC_SERVER_ERROR,
-    MSG_REQ_REDIS_DEL,                    /* redis commands - keys */
-    MSG_REQ_REDIS_EXISTS,
-    MSG_REQ_REDIS_EXPIRE,
-    MSG_REQ_REDIS_EXPIREAT,
-    MSG_REQ_REDIS_PEXPIRE,
-    MSG_REQ_REDIS_PEXPIREAT,
-    MSG_REQ_REDIS_PERSIST,
-    MSG_REQ_REDIS_PTTL,
-    MSG_REQ_REDIS_TTL,
-    MSG_REQ_REDIS_TYPE,
-    MSG_REQ_REDIS_APPEND,                 /* redis requests - string */
-    MSG_REQ_REDIS_BITCOUNT,
-    MSG_REQ_REDIS_DECR,
-    MSG_REQ_REDIS_DECRBY,
-    MSG_REQ_REDIS_DUMP,
-    MSG_REQ_REDIS_GET,
-    MSG_REQ_REDIS_GETBIT,
-    MSG_REQ_REDIS_GETRANGE,
-    MSG_REQ_REDIS_GETSET,
-    MSG_REQ_REDIS_INCR,
-    MSG_REQ_REDIS_INCRBY,
-    MSG_REQ_REDIS_INCRBYFLOAT,
-    MSG_REQ_REDIS_MGET,
-    MSG_REQ_REDIS_PSETEX,
-    MSG_REQ_REDIS_RESTORE,
-    MSG_REQ_REDIS_SET,
-    MSG_REQ_REDIS_SETBIT,
-    MSG_REQ_REDIS_SETEX,
-    MSG_REQ_REDIS_SETNX,
-    MSG_REQ_REDIS_SETRANGE,
-    MSG_REQ_REDIS_STRLEN,
-    MSG_REQ_REDIS_HDEL,                   /* redis requests - hashes */
-    MSG_REQ_REDIS_HEXISTS,
-    MSG_REQ_REDIS_HGET,
-    MSG_REQ_REDIS_HGETALL,
-    MSG_REQ_REDIS_HINCRBY,
-    MSG_REQ_REDIS_HINCRBYFLOAT,
-    MSG_REQ_REDIS_HKEYS,
-    MSG_REQ_REDIS_HLEN,
-    MSG_REQ_REDIS_HMGET,
-    MSG_REQ_REDIS_HMSET,
-    MSG_REQ_REDIS_HSET,
-    MSG_REQ_REDIS_HSETNX,
-    MSG_REQ_REDIS_HVALS,
-    MSG_REQ_REDIS_LINDEX,                 /* redis requests - lists */
-    MSG_REQ_REDIS_LINSERT,
-    MSG_REQ_REDIS_LLEN,
-    MSG_REQ_REDIS_LPOP,
-    MSG_REQ_REDIS_LPUSH,
-    MSG_REQ_REDIS_LPUSHX,
-    MSG_REQ_REDIS_LRANGE,
-    MSG_REQ_REDIS_LREM,
-    MSG_REQ_REDIS_LSET,
-    MSG_REQ_REDIS_LTRIM,
-    MSG_REQ_REDIS_RPOP,
-    MSG_REQ_REDIS_RPOPLPUSH,
-    MSG_REQ_REDIS_RPUSH,
-    MSG_REQ_REDIS_RPUSHX,
-    MSG_REQ_REDIS_SADD,                   /* redis requests - sets */
-    MSG_REQ_REDIS_SCARD,
-    MSG_REQ_REDIS_SDIFF,
-    MSG_REQ_REDIS_SDIFFSTORE,
-    MSG_REQ_REDIS_SINTER,
-    MSG_REQ_REDIS_SINTERSTORE,
-    MSG_REQ_REDIS_SISMEMBER,
-    MSG_REQ_REDIS_SMEMBERS,
-    MSG_REQ_REDIS_SMOVE,
-    MSG_REQ_REDIS_SPOP,
-    MSG_REQ_REDIS_SRANDMEMBER,
-    MSG_REQ_REDIS_SREM,
-    MSG_REQ_REDIS_SUNION,
-    MSG_REQ_REDIS_SUNIONSTORE,
-    MSG_REQ_REDIS_ZADD,                   /* redis requests - sorted sets */
-    MSG_REQ_REDIS_ZCARD,
-    MSG_REQ_REDIS_ZCOUNT,
-    MSG_REQ_REDIS_ZINCRBY,
-    MSG_REQ_REDIS_ZINTERSTORE,
-    MSG_REQ_REDIS_ZRANGE,
-    MSG_REQ_REDIS_ZRANGEBYSCORE,
-    MSG_REQ_REDIS_ZRANK,
-    MSG_REQ_REDIS_ZREM,
-    MSG_REQ_REDIS_ZREMRANGEBYRANK,
-    MSG_REQ_REDIS_ZREMRANGEBYSCORE,
-    MSG_REQ_REDIS_ZREVRANGE,
-    MSG_REQ_REDIS_ZREVRANGEBYSCORE,
-    MSG_REQ_REDIS_ZREVRANK,
-    MSG_REQ_REDIS_ZSCORE,
-    MSG_REQ_REDIS_ZUNIONSTORE,
-    MSG_REQ_REDIS_EVAL,                   /* redis requests - eval */
-    MSG_REQ_REDIS_EVALSHA,
-    MSG_RSP_REDIS_STATUS,                 /* redis response */
-    MSG_RSP_REDIS_ERROR,
-    MSG_RSP_REDIS_INTEGER,
-    MSG_RSP_REDIS_BULK,
-    MSG_RSP_REDIS_MULTIBULK,
-    MSG_SENTINEL
+    MSG_TYPE_CODEC(DEFINE_ACTION)
 } msg_type_t;
+#undef DEFINE_ACTION
+
+struct keypos {
+    uint8_t             *start;           /* key start pos */
+    uint8_t             *end;             /* key end pos */
+};
 
 struct msg {
     TAILQ_ENTRY(msg)     c_tqe;           /* link in client q */
@@ -168,6 +192,7 @@ struct msg {
 
     struct mhdr          mhdr;            /* message mbuf header */
     uint32_t             mlen;            /* message length */
+    int64_t              start_ts;        /* request start timestamp in usec */
 
     int                  state;           /* current parser state */
     uint8_t              *pos;            /* parser position marker */
@@ -176,15 +201,14 @@ struct msg {
     msg_parse_t          parser;          /* message parser */
     msg_parse_result_t   result;          /* message parsing result */
 
-    mbuf_copy_t          pre_splitcopy;   /* message pre-split copy */
-    msg_post_splitcopy_t post_splitcopy;  /* message post-split copy */
+    msg_fragment_t       fragment;        /* message fragment */
+    msg_reply_t          reply;           /* gen message reply (example: ping) */
     msg_coalesce_t       pre_coalesce;    /* message pre-coalesce */
     msg_coalesce_t       post_coalesce;   /* message post-coalesce */
 
     msg_type_t           type;            /* message type */
 
-    uint8_t              *key_start;      /* key start */
-    uint8_t              *key_end;        /* key end */
+    struct array         *keys;           /* array of keypos, for req */
 
     uint32_t             vlen;            /* value length (memcache) */
     uint8_t              *end;            /* end marker (memcache) */
@@ -198,7 +222,9 @@ struct msg {
 
     struct msg           *frag_owner;     /* owner of fragment message */
     uint32_t             nfrag;           /* # fragment */
+    uint32_t             nfrag_done;      /* # fragment done */
     uint64_t             frag_id;         /* id of fragmented message */
+    struct msg           **frag_seq;      /* sequence of fragment message, map from keys to fragments*/
 
     err_t                err;             /* errno on error? */
     unsigned             error:1;         /* error? */
@@ -206,10 +232,9 @@ struct msg {
     unsigned             request:1;       /* request? or response? */
     unsigned             quit:1;          /* quit request? */
     unsigned             noreply:1;       /* noreply? */
+    unsigned             noforward:1;     /* not need forward (example: ping) */
     unsigned             done:1;          /* done? */
     unsigned             fdone:1;         /* all fragments are done? */
-    unsigned             first_fragment:1;/* first fragment? */
-    unsigned             last_fragment:1; /* last fragment? */
     unsigned             swallow:1;       /* swallow response? */
     unsigned             redis:1;         /* redis? */
 };
@@ -222,13 +247,20 @@ void msg_tmo_delete(struct msg *msg);
 
 void msg_init(void);
 void msg_deinit(void);
+struct string *msg_type_string(msg_type_t type);
 struct msg *msg_get(struct conn *conn, bool request, bool redis);
 void msg_put(struct msg *msg);
 struct msg *msg_get_error(bool redis, err_t err);
-void msg_dump(struct msg *msg);
+void msg_dump(struct msg *msg, int level);
 bool msg_empty(struct msg *msg);
 rstatus_t msg_recv(struct context *ctx, struct conn *conn);
 rstatus_t msg_send(struct context *ctx, struct conn *conn);
+uint64_t msg_gen_frag_id(void);
+uint32_t msg_backend_idx(struct msg *msg, uint8_t *key, uint32_t keylen);
+struct mbuf *msg_ensure_mbuf(struct msg *msg, size_t len);
+rstatus_t msg_append(struct msg *msg, uint8_t *pos, size_t n);
+rstatus_t msg_prepend(struct msg *msg, uint8_t *pos, size_t n);
+rstatus_t msg_prepend_format(struct msg *msg, const char *fmt, ...);
 
 struct msg *req_get(struct conn *conn);
 void req_put(struct msg *msg);

data/ext/nutcracker/src/nc_proxy.c

@@ -276,24 +276,54 @@ proxy_accept(struct context *ctx, struct conn *p)
                 continue;
             }
 
-            if (errno == EAGAIN || errno == EWOULDBLOCK) {
+            if (errno == EAGAIN || errno == EWOULDBLOCK || errno == ECONNABORTED) {
                 log_debug(LOG_VERB, "accept on p %d not ready - eagain", p->sd);
                 p->recv_ready = 0;
                 return NC_OK;
             }
 
-            /*
-             * FIXME: On EMFILE or ENFILE mask out IN event on the proxy; mask
-             * it back in when some existing connection gets closed
+            /* 
+             * Workaround of https://github.com/twitter/twemproxy/issues/97
+             *
+             * We should never reach here because the check for conn_ncurr_cconn()
+             * against ctx->max_ncconn should catch this earlier in the cycle.
+             * If we reach here ignore EMFILE/ENFILE, return NC_OK will enable
+             * the server continue to run instead of close the server socket
+             *
+             * The right solution however, is on EMFILE/ENFILE to mask out IN
+             * event on the proxy and mask it back in when some existing
+             * connections gets closed
              */
+            if (errno == EMFILE || errno == ENFILE) {
+                log_debug(LOG_CRIT, "accept on p %d with max fds %"PRIu32" "
+                          "used connections %"PRIu32" max client connections %"PRIu32" "
+                          "curr client connections %"PRIu32" failed: %s",
+                          p->sd, ctx->max_nfd, conn_ncurr_conn(),
+                          ctx->max_ncconn, conn_ncurr_cconn(), strerror(errno));
+
+                p->recv_ready = 0;
+
+                return NC_OK;
+            }
 
             log_error("accept on p %d failed: %s", p->sd, strerror(errno));
+
             return NC_ERROR;
         }
 
         break;
     }
 
+    if (conn_ncurr_cconn() >= ctx->max_ncconn) {
+        log_debug(LOG_CRIT, "client connections %"PRIu32" exceed limit %"PRIu32,
+                  conn_ncurr_cconn(), ctx->max_ncconn);
+        status = close(sd);
+        if (status < 0) {
+            log_error("close c %d failed, ignored: %s", sd, strerror(errno));
+        }
+        return NC_OK;
+    }
+
     c = conn_get(p->owner, true, p->redis);
     if (c == NULL) {
         log_error("get conn for c %d from p %d failed: %s", sd, p->sd,

data/ext/nutcracker/src/nc_request.c

@@ -29,10 +29,73 @@ req_get(struct conn *conn)
     if (msg == NULL) {
         conn->err = errno;
     }
-
     return msg;
 }
 
+static void
+req_log(struct msg *req)
+{
+    struct msg *rsp;           /* peer message (response) */
+    int64_t req_time;          /* time cost for this request */
+    char *peer_str;            /* peer client ip:port */
+    uint32_t req_len, rsp_len; /* request and response length */
+    struct string *req_type;   /* request type string */
+    struct keypos *kpos;
+
+    if (log_loggable(LOG_NOTICE) == 0) {
+        return;
+    }
+
+    /* a fragment? */
+    if (req->frag_id != 0 && req->frag_owner != req) {
+        return;
+    }
+
+    /* conn close normally? */
+    if (req->mlen == 0) {
+        return;
+    }
+    /*
+     * there is a race scenario where a requests comes in, the log level is not LOG_NOTICE,
+     * and before the response arrives you modify the log level to LOG_NOTICE
+     * using SIGTTIN OR SIGTTOU, then req_log() wouldn't have msg->start_ts set
+     */
+    if (req->start_ts == 0) {
+        return;
+    }
+
+    req_time = nc_usec_now() - req->start_ts;
+
+    rsp = req->peer;
+    req_len = req->mlen;
+    rsp_len = (rsp != NULL) ? rsp->mlen : 0;
+
+    if (array_n(req->keys) < 1) {
+        return;
+    }
+
+    kpos = array_get(req->keys, 0);
+    if (kpos->end != NULL) {
+        *(kpos->end) = '\0';
+    }
+
+    /*
+     * FIXME: add backend addr here
+     * Maybe we can store addrstr just like server_pool in conn struct
+     * when connections are resolved
+     */
+    peer_str = nc_unresolve_peer_desc(req->owner->sd);
+
+    req_type = msg_type_string(req->type);
+
+    log_debug(LOG_NOTICE, "req %"PRIu64" done on c %d req_time %"PRIi64".%03"PRIi64
+              " msec type %.*s narg %"PRIu32" req_len %"PRIu32" rsp_len %"PRIu32
+              " key0 '%s' peer '%s' done %d error %d",
+              req->id, req->owner->sd, req_time / 1000, req_time % 1000,
+              req_type->len, req_type->data, req->narg, req_len, rsp_len,
+              kpos->start, peer_str, req->done, req->error);
+}
+
 void
 req_put(struct msg *msg)
 {
@@ -40,6 +103,8 @@ req_put(struct msg *msg)
 
     ASSERT(msg->request);
 
+    req_log(msg);
+
     pmsg = msg->peer;
     if (pmsg != NULL) {
         ASSERT(!pmsg->request && pmsg->peer == msg);
@@ -84,6 +149,10 @@ req_done(struct conn *conn, struct msg *msg)
         return true;
     }
 
+    if (msg->nfrag_done < msg->nfrag) {
+        return false;
+    }
+
     /* check all fragments of the given request vector are done */
 
     for (pmsg = msg, cmsg = TAILQ_PREV(msg, msg_tqh, c_tqe);
@@ -104,10 +173,6 @@ req_done(struct conn *conn, struct msg *msg)
         }
     }
 
-    if (!pmsg->last_fragment) {
-        return false;
-    }
-
     /*
      * At this point, all the fragments including the last fragment have
      * been received.
@@ -117,7 +182,7 @@ req_done(struct conn *conn, struct msg *msg)
      */
 
     msg->fdone = 1;
-    nfragment = 1;
+    nfragment = 0;
 
     for (pmsg = msg, cmsg = TAILQ_PREV(msg, msg_tqh, c_tqe);
          cmsg != NULL && cmsg->frag_id == id;
@@ -143,6 +208,7 @@ req_done(struct conn *conn, struct msg *msg)
     return true;
 }
 
+
 /*
  * Return true if request is in error, false otherwise
  *
@@ -360,6 +426,26 @@ req_recv_next(struct context *ctx, struct conn *conn, bool alloc)
     return msg;
 }
 
+static rstatus_t
+req_make_reply(struct context *ctx, struct conn *conn, struct msg *req)
+{
+    struct msg *msg;
+
+    msg = msg_get(conn, true, conn->redis); /* replay */
+    if (msg == NULL) {
+        conn->err = errno;
+        return NC_ENOMEM;
+    }
+
+    req->peer = msg;
+    msg->peer = req;
+    msg->request = 0;
+
+    req->done = 1;
+    conn->enqueue_outq(ctx, conn, req);
+    return NC_OK;
+}
+
 static bool
 req_filter(struct context *ctx, struct conn *conn, struct msg *msg)
 {
@@ -436,6 +522,7 @@ req_forward(struct context *ctx, struct conn *c_conn, struct msg *msg)
     struct server_pool *pool;
     uint8_t *key;
     uint32_t keylen;
+    struct keypos *kpos;
 
     ASSERT(c_conn->client && !c_conn->proxy);
 
@@ -445,32 +532,11 @@ req_forward(struct context *ctx, struct conn *c_conn, struct msg *msg)
     }
 
     pool = c_conn->owner;
-    key = NULL;
-    keylen = 0;
 
-    /*
-     * If hash_tag: is configured for this server pool, we use the part of
-     * the key within the hash tag as an input to the distributor. Otherwise
-     * we use the full key
-     */
-    if (!string_empty(&pool->hash_tag)) {
-        struct string *tag = &pool->hash_tag;
-        uint8_t *tag_start, *tag_end;
-
-        tag_start = nc_strchr(msg->key_start, msg->key_end, tag->data[0]);
-        if (tag_start != NULL) {
-            tag_end = nc_strchr(tag_start + 1, msg->key_end, tag->data[1]);
-            if (tag_end != NULL) {
-                key = tag_start + 1;
-                keylen = (uint32_t)(tag_end - key);
-            }
-        }
-    }
-
-    if (keylen == 0) {
-        key = msg->key_start;
-        keylen = (uint32_t)(msg->key_end - msg->key_start);
-    }
+    ASSERT(array_n(msg->keys) > 0);
+    kpos = array_get(msg->keys, 0);
+    key = kpos->start;
+    keylen = (uint32_t)(kpos->end - kpos->start);
 
     s_conn = server_pool_conn(ctx, c_conn->owner, key, keylen);
     if (s_conn == NULL) {
@@ -501,6 +567,12 @@ void
 req_recv_done(struct context *ctx, struct conn *conn, struct msg *msg,
               struct msg *nmsg)
 {
+    rstatus_t status;
+    struct server_pool *pool;
+    struct msg_tqh frag_msgq;
+    struct msg *sub_msg;
+    struct msg *tmsg; 			/* tmp next message */
+
     ASSERT(conn->client && !conn->proxy);
     ASSERT(msg->request);
     ASSERT(msg->owner == conn);
@@ -514,7 +586,61 @@ req_recv_done(struct context *ctx, struct conn *conn, struct msg *msg,
         return;
     }
 
-    req_forward(ctx, conn, msg);
+    if (msg->noforward) {
+        status = req_make_reply(ctx, conn, msg);
+        if (status != NC_OK) {
+            conn->err = errno;
+            return;
+        }
+
+        status = msg->reply(msg);
+        if (status != NC_OK) {
+            conn->err = errno;
+            return;
+        }
+
+        status = event_add_out(ctx->evb, conn);
+        if (status != NC_OK) {
+            conn->err = errno;
+        }
+
+        return;
+    }
+
+    /* do fragment */
+    pool = conn->owner;
+    TAILQ_INIT(&frag_msgq);
+    status = msg->fragment(msg, pool->ncontinuum, &frag_msgq);
+    if (status != NC_OK) {
+        if (!msg->noreply) {
+            conn->enqueue_outq(ctx, conn, msg);
+        }
+        req_forward_error(ctx, conn, msg);
+    }
+
+    /* if no fragment happened */
+    if (TAILQ_EMPTY(&frag_msgq)) {
+        req_forward(ctx, conn, msg);
+        return;
+    }
+
+    status = req_make_reply(ctx, conn, msg);
+    if (status != NC_OK) {
+        if (!msg->noreply) {
+            conn->enqueue_outq(ctx, conn, msg);
+        }
+        req_forward_error(ctx, conn, msg);
+    }
+
+    for (sub_msg = TAILQ_FIRST(&frag_msgq); sub_msg != NULL; sub_msg = tmsg) {
+        tmsg = TAILQ_NEXT(sub_msg, m_tqe);
+
+        TAILQ_REMOVE(&frag_msgq, sub_msg, m_tqe);
+        req_forward(ctx, conn, sub_msg);
+    }
+
+    ASSERT(TAILQ_EMPTY(&frag_msgq));
+    return;
 }
 
 struct msg *