nutcracker 0.4.0.16 → 0.4.1.18

data/ext/nutcracker/src/hashkit/Makefile.in

@@ -1,7 +1,7 @@
-# Makefile.in generated by automake 1.14.1 from Makefile.am.
+# Makefile.in generated by automake 1.15 from Makefile.am.
 # @configure_input@
 
-# Copyright (C) 1994-2013 Free Software Foundation, Inc.
+# Copyright (C) 1994-2014 Free Software Foundation, Inc.
 
 # This Makefile.in is free software; the Free Software Foundation
 # gives unlimited permission to copy and/or distribute it,
@@ -16,7 +16,17 @@
 
 
 VPATH = @srcdir@
-am__is_gnu_make = test -n '$(MAKEFILE_LIST)' && test -n '$(MAKELEVEL)'
+am__is_gnu_make = { \
+  if test -z '$(MAKELEVEL)'; then \
+    false; \
+  elif test -n '$(MAKE_HOST)'; then \
+    true; \
+  elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \
+    true; \
+  else \
+    false; \
+  fi; \
+}
 am__make_running_with_option = \
   case $${target_option-} in \
       ?) ;; \
@@ -80,8 +90,6 @@ POST_UNINSTALL = :
 build_triplet = @build@
 host_triplet = @host@
 subdir = src/hashkit
-DIST_COMMON = $(srcdir)/Makefile.in $(srcdir)/Makefile.am \
-	$(top_srcdir)/config/depcomp $(noinst_HEADERS)
 ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
 am__aclocal_m4_deps = $(top_srcdir)/m4/libtool.m4 \
 	$(top_srcdir)/m4/ltoptions.m4 $(top_srcdir)/m4/ltsugar.m4 \
@@ -89,6 +97,8 @@ am__aclocal_m4_deps = $(top_srcdir)/m4/libtool.m4 \
 	$(top_srcdir)/configure.ac
 am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
 	$(ACLOCAL_M4)
+DIST_COMMON = $(srcdir)/Makefile.am $(noinst_HEADERS) \
+	$(am__DIST_COMMON)
 mkinstalldirs = $(install_sh) -d
 CONFIG_HEADER = $(top_builddir)/config.h
 CONFIG_CLEAN_FILES =
@@ -172,6 +182,7 @@ am__define_uniq_tagged_files = \
   done | $(am__uniquify_input)`
 ETAGS = etags
 CTAGS = ctags
+am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/config/depcomp
 DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
 ACLOCAL = @ACLOCAL@
 AMTAR = @AMTAR@
@@ -216,6 +227,7 @@ LIBTOOL = @LIBTOOL@
 LIPO = @LIPO@
 LN_S = @LN_S@
 LTLIBOBJS = @LTLIBOBJS@
+LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@
 MAKEINFO = @MAKEINFO@
 MANIFEST_TOOL = @MANIFEST_TOOL@
 MKDIR_P = @MKDIR_P@
@@ -327,7 +339,6 @@ $(srcdir)/Makefile.in:  $(srcdir)/Makefile.am  $(am__configure_deps)
 	echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/hashkit/Makefile'; \
 	$(am__cd) $(top_srcdir) && \
 	  $(AUTOMAKE) --foreign src/hashkit/Makefile
-.PRECIOUS: Makefile
 Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
 	@case '$?' in \
 	  *config.status*) \
@@ -603,6 +614,8 @@ uninstall-am:
 	mostlyclean-generic mostlyclean-libtool pdf pdf-am ps ps-am \
 	tags tags-am uninstall uninstall-am
 
+.PRECIOUS: Makefile
+
 
 # Tell versions [3.59,3.63) of GNU make to not export all variables.
 # Otherwise a system limit (for SysV at least) may be exceeded.

data/ext/nutcracker/src/hashkit/nc_jenkins.c

@@ -106,7 +106,7 @@ hash_jenkins(const char *key, size_t length)
      * rest of the string.  Every machine with memory protection I've seen
      * does it on word boundaries, so is OK with this.  But VALGRIND will
      * still catch it and complain.  The masking trick does make the hash
-     * noticably faster for short strings (like English words).
+     * noticeably faster for short strings (like English words).
      */
     switch(length)
     {

data/ext/nutcracker/src/hashkit/nc_ketama.c

@@ -162,10 +162,10 @@ ketama_update(struct server_pool *pool)
         pointer_per_server = (uint32_t) ((floorf((float) (pct * KETAMA_POINTS_PER_SERVER / 4 * (float)nlive_server + 0.0000000001))) * 4);
         pointer_per_hash = 4;
 
-        log_debug(LOG_VERB, "%.*s:%"PRIu16" weight %"PRIu32" of %"PRIu32" "
+        log_debug(LOG_VERB, "%.*s weight %"PRIu32" of %"PRIu32" "
                   "pct %0.5f points per server %"PRIu32"",
-                  server->name.len, server->name.data, server->port,
-                  server->weight, total_weight, pct, pointer_per_server);
+                  server->name.len, server->name.data, server->weight,
+                  total_weight, pct, pointer_per_server);
 
         for (pointer_index = 1;
              pointer_index <= pointer_per_server / pointer_per_hash;

data/ext/nutcracker/src/nc.c

@@ -214,7 +214,7 @@ nc_show_usage(void)
         "  -d, --daemonize        : run as a daemon" CRLF
         "  -D, --describe-stats   : print stats description and exit");
     log_stderr(
-        "  -v, --verbosity=N      : set logging level (default: %d, min: %d, max: %d)" CRLF
+        "  -v, --verbose=N        : set logging level (default: %d, min: %d, max: %d)" CRLF
         "  -o, --output=S         : set logging file (default: %s)" CRLF
         "  -c, --conf-file=S      : set configuration file (default: %s)" CRLF
         "  -s, --stats-port=N     : set stats monitoring port (default: %d)" CRLF

data/ext/nutcracker/src/nc_conf.c

@@ -74,6 +74,14 @@ static struct command conf_commands[] = {
       conf_set_bool,
       offsetof(struct conf_pool, redis) },
 
+    { string("redis_auth"),
+      conf_set_string,
+      offsetof(struct conf_pool, redis_auth) },
+
+    { string("redis_db"),
+      conf_set_num,
+      offsetof(struct conf_pool, redis_db) },
+
     { string("preconnect"),
       conf_set_bool,
       offsetof(struct conf_pool, preconnect) },
@@ -106,6 +114,7 @@ conf_server_init(struct conf_server *cs)
 {
     string_init(&cs->pname);
     string_init(&cs->name);
+    string_init(&cs->addrstr);
     cs->port = 0;
     cs->weight = 0;
 
@@ -121,6 +130,7 @@ conf_server_deinit(struct conf_server *cs)
 {
     string_deinit(&cs->pname);
     string_deinit(&cs->name);
+    string_deinit(&cs->addrstr);
     cs->valid = 0;
     log_debug(LOG_VVERB, "deinit conf server %p", cs);
 }
@@ -142,12 +152,11 @@ conf_server_each_transform(void *elem, void *data)
 
     s->pname = cs->pname;
     s->name = cs->name;
+    s->addrstr = cs->addrstr;
     s->port = (uint16_t)cs->port;
     s->weight = (uint32_t)cs->weight;
 
-    s->family = cs->info.family;
-    s->addrlen = cs->info.addrlen;
-    s->addr = (struct sockaddr *)&cs->info.addr;
+    nc_memcpy(&s->info, &cs->info, sizeof(cs->info));
 
     s->ns_conn_q = 0;
     TAILQ_INIT(&s->s_conn_q);
@@ -170,6 +179,7 @@ conf_pool_init(struct conf_pool *cp, struct string *name)
 
     string_init(&cp->listen.pname);
     string_init(&cp->listen.name);
+    string_init(&cp->redis_auth);
     cp->listen.port = 0;
     memset(&cp->listen.info, 0, sizeof(cp->listen.info));
     cp->listen.valid = 0;
@@ -184,6 +194,7 @@ conf_pool_init(struct conf_pool *cp, struct string *name)
     cp->client_connections = CONF_UNSET_NUM;
 
     cp->redis = CONF_UNSET_NUM;
+    cp->redis_db = CONF_UNSET_NUM;
     cp->preconnect = CONF_UNSET_NUM;
     cp->auto_eject_hosts = CONF_UNSET_NUM;
     cp->server_connections = CONF_UNSET_NUM;
@@ -219,6 +230,10 @@ conf_pool_deinit(struct conf_pool *cp)
     string_deinit(&cp->listen.pname);
     string_deinit(&cp->listen.name);
 
+    if (cp->redis_auth.len > 0) {
+        string_deinit(&cp->redis_auth);
+    }
+
     while (array_n(&cp->server) != 0) {
         conf_server_deinit(array_pop(&cp->server));
     }
@@ -258,9 +273,8 @@ conf_pool_each_transform(void *elem, void *data)
     sp->addrstr = cp->listen.pname;
     sp->port = (uint16_t)cp->listen.port;
 
-    sp->family = cp->listen.info.family;
-    sp->addrlen = cp->listen.info.addrlen;
-    sp->addr = (struct sockaddr *)&cp->listen.info.addr;
+    nc_memcpy(&sp->info, &cp->listen.info, sizeof(cp->listen.info));
+    sp->perm = cp->listen.perm;
 
     sp->key_hash_type = cp->hash;
     sp->key_hash = hash_algos[cp->hash];
@@ -270,9 +284,12 @@ conf_pool_each_transform(void *elem, void *data)
     sp->redis = cp->redis ? 1 : 0;
     sp->timeout = cp->timeout;
     sp->backlog = cp->backlog;
+    sp->redis_db = cp->redis_db;
 
-    sp->client_connections = (uint32_t)cp->client_connections;
+    sp->redis_auth = cp->redis_auth;
+    sp->require_auth = cp->redis_auth.len > 0 ? 1 : 0;
 
+    sp->client_connections = (uint32_t)cp->client_connections;
     sp->server_connections = (uint32_t)cp->server_connections;
     sp->server_retry_timeout = (int64_t)cp->server_retry_timeout * 1000LL;
     sp->server_failure_limit = (uint32_t)cp->server_failure_limit;
@@ -1161,7 +1178,7 @@ conf_validate_server(struct conf *cf, struct conf_pool *cp)
 
         if (string_compare(&cs1->name, &cs2->name) == 0) {
             log_error("conf: pool '%.*s' has servers with same name '%.*s'",
-                      cp->name.len, cp->name.data, cs1->name.len, 
+                      cp->name.len, cp->name.data, cs1->name.len,
                       cs1->name.data);
             valid = false;
             break;
@@ -1211,6 +1228,10 @@ conf_validate_pool(struct conf *cf, struct conf_pool *cp)
         cp->redis = CONF_DEFAULT_REDIS;
     }
 
+    if (cp->redis_db == CONF_UNSET_NUM) {
+        cp->redis_db = CONF_DEFAULT_REDIS_DB;
+    }
+
     if (cp->preconnect == CONF_UNSET_NUM) {
         cp->preconnect = CONF_DEFAULT_PRECONNECT;
     }
@@ -1234,6 +1255,11 @@ conf_validate_pool(struct conf *cf, struct conf_pool *cp)
         cp->server_failure_limit = CONF_DEFAULT_SERVER_FAILURE_LIMIT;
     }
 
+    if (!cp->redis && cp->redis_auth.len > 0) {
+        log_error("conf: directive \"redis_auth:\" is only valid for a redis pool");
+        return NC_ERROR;
+    }
+
     status = conf_validate_server(cf, cp);
     if (status != NC_OK) {
         return status;
@@ -1420,8 +1446,32 @@ conf_set_listen(struct conf *cf, struct command *cmd, void *conf)
     }
 
     if (value->data[0] == '/') {
-        name = value->data;
-        namelen = value->len;
+        uint8_t *q, *start, *perm;
+        uint32_t permlen;
+
+
+        /* parse "socket_path permissions" from the end */
+        p = value->data + value->len -1;
+        start = value->data;
+        q = nc_strrchr(p, start, ' ');
+        if (q == NULL) {
+            /* no permissions field, so use defaults */
+            name = value->data;
+            namelen = value->len;
+        } else {
+            perm = q + 1;
+            permlen = (uint32_t)(p - perm + 1);
+
+            p = q - 1;
+            name = start;
+            namelen = (uint32_t)(p - start + 1);
+
+            errno = 0;
+            field->perm = (mode_t)strtol((char *)perm, NULL, 8);
+            if (errno || field->perm > 0777) {
+                return "has an invalid file permission in \"socket_path permission\" format string";
+            }
+        }
     } else {
         uint8_t *q, *start, *port;
         uint32_t portlen;
@@ -1473,10 +1523,8 @@ conf_add_server(struct conf *cf, struct command *cmd, void *conf)
     uint8_t *p, *q, *start;
     uint8_t *pname, *addr, *port, *weight, *name;
     uint32_t k, delimlen, pnamelen, addrlen, portlen, weightlen, namelen;
-    struct string address;
     char delim[] = " ::";
 
-    string_init(&address);
     p = conf;
     a = (struct array *)(p + cmd->offset);
 
@@ -1588,18 +1636,18 @@ conf_add_server(struct conf *cf, struct command *cmd, void *conf)
         return CONF_ERROR;
     }
 
-    status = string_copy(&address, addr, addrlen);
+    status = string_copy(&field->addrstr, addr, addrlen);
     if (status != NC_OK) {
         return CONF_ERROR;
     }
 
-    status = nc_resolve(&address, field->port, &field->info);
-    if (status != NC_OK) {
-        string_deinit(&address);
-        return CONF_ERROR;
-    }
+    /*
+     * The address resolution of the backend server hostname is lazy.
+     * The resolution occurs when a new connection to the server is
+     * created, which could either be the first time or every time
+     * the server gets re-added to the pool after an auto ejection
+     */
 
-    string_deinit(&address);
     field->valid = 1;
 
     return CONF_OK;

data/ext/nutcracker/src/nc_conf.h

@@ -47,6 +47,7 @@
 #define CONF_DEFAULT_LISTEN_BACKLOG          512
 #define CONF_DEFAULT_CLIENT_CONNECTIONS      0
 #define CONF_DEFAULT_REDIS                   false
+#define CONF_DEFAULT_REDIS_DB                0
 #define CONF_DEFAULT_PRECONNECT              false
 #define CONF_DEFAULT_AUTO_EJECT_HOSTS        false
 #define CONF_DEFAULT_SERVER_RETRY_TIMEOUT    30 * 1000      /* in msec */
@@ -55,16 +56,18 @@
 #define CONF_DEFAULT_KETAMA_PORT             11211
 
 struct conf_listen {
-    struct string   pname;   /* listen: as "name:port" */
-    struct string   name;    /* name */
+    struct string   pname;   /* listen: as "hostname:port" */
+    struct string   name;    /* hostname:port */
     int             port;    /* port */
+    mode_t          perm;    /* socket permissions */
     struct sockinfo info;    /* listen socket info */
     unsigned        valid:1; /* valid? */
 };
 
 struct conf_server {
-    struct string   pname;      /* server: as "name:port:weight" */
-    struct string   name;       /* name */
+    struct string   pname;      /* server: as "hostname:port:weight" */
+    struct string   name;       /* hostname:port or [name] */
+    struct string   addrstr;    /* hostname */
     int             port;       /* port */
     int             weight;     /* weight */
     struct sockinfo info;       /* connect socket info */
@@ -81,6 +84,8 @@ struct conf_pool {
     int                backlog;               /* backlog: */
     int                client_connections;    /* client_connections: */
     int                redis;                 /* redis: */
+    struct string      redis_auth;            /* redis_auth: redis auth password (matches requirepass on redis) */
+    int                redis_db;              /* redis_db: redis db */
     int                preconnect;            /* preconnect: */
     int                auto_eject_hosts;      /* auto_eject_hosts: */
     int                server_connections;    /* server_connections: */

data/ext/nutcracker/src/nc_connection.c

@@ -156,6 +156,7 @@ _conn_get(void)
     conn->eof = 0;
     conn->done = 0;
     conn->redis = 0;
+    conn->authenticated = 0;
 
     ntotal_conn++;
     ncurr_conn++;
@@ -201,7 +202,9 @@ conn_get(void *owner, bool client, bool redis)
         conn->dequeue_inq = NULL;
         conn->enqueue_outq = req_client_enqueue_omsgq;
         conn->dequeue_outq = req_client_dequeue_omsgq;
-        
+        conn->post_connect = NULL;
+        conn->swallow_msg = NULL;
+
         ncurr_cconn++;
     } else {
         /*
@@ -226,10 +229,16 @@ conn_get(void *owner, bool client, bool redis)
         conn->dequeue_inq = req_server_dequeue_imsgq;
         conn->enqueue_outq = req_server_enqueue_omsgq;
         conn->dequeue_outq = req_server_dequeue_omsgq;
+        if (redis) {
+          conn->post_connect = redis_post_connect;
+          conn->swallow_msg = redis_swallow_msg;
+        } else {
+          conn->post_connect = memcache_post_connect;
+          conn->swallow_msg = memcache_swallow_msg;
+        }
     }
 
     conn->ref(conn, owner);
-
     log_debug(LOG_VVERB, "get conn %p client %d", conn, conn->client);
 
     return conn;
@@ -438,3 +447,27 @@ conn_ncurr_cconn(void)
 {
     return ncurr_cconn;
 }
+
+/*
+ * Returns true if the connection is authenticated or doesn't require
+ * authentication, otherwise return false
+ */
+bool
+conn_authenticated(struct conn *conn)
+{
+    struct server_pool *pool;
+
+    ASSERT(!conn->proxy);
+
+    pool = conn->client ? conn->owner : ((struct server *)conn->owner)->owner;
+
+    if (!pool->require_auth) {
+        return true;
+    }
+
+    if (!conn->authenticated) {
+        return false;
+    }
+
+    return true;
+}

data/ext/nutcracker/src/nc_connection.h

@@ -35,55 +35,60 @@ typedef void (*conn_ref_t)(struct conn *, void *);
 typedef void (*conn_unref_t)(struct conn *);
 
 typedef void (*conn_msgq_t)(struct context *, struct conn *, struct msg *);
+typedef void (*conn_post_connect_t)(struct context *ctx, struct conn *, struct server *server);
+typedef void (*conn_swallow_msg_t)(struct conn *, struct msg *, struct msg *);
 
 struct conn {
-    TAILQ_ENTRY(conn)  conn_tqe;      /* link in server_pool / server / free q */
-    void               *owner;        /* connection owner - server_pool / server */
-
-    int                sd;            /* socket descriptor */
-    int                family;        /* socket address family */
-    socklen_t          addrlen;       /* socket length */
-    struct sockaddr    *addr;         /* socket address (ref in server or server_pool) */
-
-    struct msg_tqh     imsg_q;        /* incoming request Q */
-    struct msg_tqh     omsg_q;        /* outstanding request Q */
-    struct msg         *rmsg;         /* current message being rcvd */
-    struct msg         *smsg;         /* current message being sent */
-
-    conn_recv_t        recv;          /* recv (read) handler */
-    conn_recv_next_t   recv_next;     /* recv next message handler */
-    conn_recv_done_t   recv_done;     /* read done handler */
-    conn_send_t        send;          /* send (write) handler */
-    conn_send_next_t   send_next;     /* write next message handler */
-    conn_send_done_t   send_done;     /* write done handler */
-    conn_close_t       close;         /* close handler */
-    conn_active_t      active;        /* active? handler */
-
-    conn_ref_t         ref;           /* connection reference handler */
-    conn_unref_t       unref;         /* connection unreference handler */
-
-    conn_msgq_t        enqueue_inq;   /* connection inq msg enqueue handler */
-    conn_msgq_t        dequeue_inq;   /* connection inq msg dequeue handler */
-    conn_msgq_t        enqueue_outq;  /* connection outq msg enqueue handler */
-    conn_msgq_t        dequeue_outq;  /* connection outq msg dequeue handler */
-
-    size_t             recv_bytes;    /* received (read) bytes */
-    size_t             send_bytes;    /* sent (written) bytes */
-
-    uint32_t           events;        /* connection io events */
-    err_t              err;           /* connection errno */
-    unsigned           recv_active:1; /* recv active? */
-    unsigned           recv_ready:1;  /* recv ready? */
-    unsigned           send_active:1; /* send active? */
-    unsigned           send_ready:1;  /* send ready? */
-
-    unsigned           client:1;      /* client? or server? */
-    unsigned           proxy:1;       /* proxy? */
-    unsigned           connecting:1;  /* connecting? */
-    unsigned           connected:1;   /* connected? */
-    unsigned           eof:1;         /* eof? aka passive close? */
-    unsigned           done:1;        /* done? aka close? */
-    unsigned           redis:1;       /* redis? */
+    TAILQ_ENTRY(conn)   conn_tqe;        /* link in server_pool / server / free q */
+    void                *owner;          /* connection owner - server_pool / server */
+
+    int                 sd;              /* socket descriptor */
+    int                 family;          /* socket address family */
+    socklen_t           addrlen;         /* socket length */
+    struct sockaddr     *addr;           /* socket address (ref in server or server_pool) */
+
+    struct msg_tqh      imsg_q;          /* incoming request Q */
+    struct msg_tqh      omsg_q;          /* outstanding request Q */
+    struct msg          *rmsg;           /* current message being rcvd */
+    struct msg          *smsg;           /* current message being sent */
+
+    conn_recv_t         recv;            /* recv (read) handler */
+    conn_recv_next_t    recv_next;       /* recv next message handler */
+    conn_recv_done_t    recv_done;       /* read done handler */
+    conn_send_t         send;            /* send (write) handler */
+    conn_send_next_t    send_next;       /* write next message handler */
+    conn_send_done_t    send_done;       /* write done handler */
+    conn_close_t        close;           /* close handler */
+    conn_active_t       active;          /* active? handler */
+    conn_post_connect_t post_connect;    /* post connect handler */
+    conn_swallow_msg_t  swallow_msg;     /* react on messages to be swallowed */
+
+    conn_ref_t          ref;             /* connection reference handler */
+    conn_unref_t        unref;           /* connection unreference handler */
+
+    conn_msgq_t         enqueue_inq;     /* connection inq msg enqueue handler */
+    conn_msgq_t         dequeue_inq;     /* connection inq msg dequeue handler */
+    conn_msgq_t         enqueue_outq;    /* connection outq msg enqueue handler */
+    conn_msgq_t         dequeue_outq;    /* connection outq msg dequeue handler */
+
+    size_t              recv_bytes;      /* received (read) bytes */
+    size_t              send_bytes;      /* sent (written) bytes */
+
+    uint32_t            events;          /* connection io events */
+    err_t               err;             /* connection errno */
+    unsigned            recv_active:1;   /* recv active? */
+    unsigned            recv_ready:1;    /* recv ready? */
+    unsigned            send_active:1;   /* send active? */
+    unsigned            send_ready:1;    /* send ready? */
+
+    unsigned            client:1;        /* client? or server? */
+    unsigned            proxy:1;         /* proxy? */
+    unsigned            connecting:1;    /* connecting? */
+    unsigned            connected:1;     /* connected? */
+    unsigned            eof:1;           /* eof? aka passive close? */
+    unsigned            done:1;          /* done? aka close? */
+    unsigned            redis:1;         /* redis? */
+    unsigned            authenticated:1; /* authenticated? */
 };
 
 TAILQ_HEAD(conn_tqh, conn);
@@ -99,5 +104,6 @@ void conn_deinit(void);
 uint32_t conn_ncurr_conn(void);
 uint64_t conn_ntotal_conn(void);
 uint32_t conn_ncurr_cconn(void);
+bool conn_authenticated(struct conn *conn);
 
 #endif