nutcracker 0.2.3

data/ext/nutcracker/src/nc_connection.h

@@ -0,0 +1,99 @@
+/*
+ * twemproxy - A fast and lightweight proxy for memcached protocol.
+ * Copyright (C) 2011 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _NC_CONNECTION_H_
+#define _NC_CONNECTION_H_
+
+#include <nc_core.h>
+
+typedef rstatus_t (*conn_recv_t)(struct context *, struct conn*);
+typedef struct msg* (*conn_recv_next_t)(struct context *, struct conn *, bool);
+typedef void (*conn_recv_done_t)(struct context *, struct conn *, struct msg *, struct msg *);
+
+typedef rstatus_t (*conn_send_t)(struct context *, struct conn*);
+typedef struct msg* (*conn_send_next_t)(struct context *, struct conn *);
+typedef void (*conn_send_done_t)(struct context *, struct conn *, struct msg *);
+
+typedef void (*conn_close_t)(struct context *, struct conn *);
+typedef bool (*conn_active_t)(struct conn *);
+
+typedef void (*conn_ref_t)(struct conn *, void *);
+typedef void (*conn_unref_t)(struct conn *);
+
+typedef void (*conn_msgq_t)(struct context *, struct conn *, struct msg *);
+
+struct conn {
+    TAILQ_ENTRY(conn)  conn_tqe;      /* link in server_pool / server / free q */
+    void               *owner;        /* connection owner - server_pool / server */
+
+    int                sd;            /* socket descriptor */
+    int                family;        /* socket address family */
+    socklen_t          addrlen;       /* socket length */
+    struct sockaddr    *addr;         /* socket address (ref in server or server_pool) */
+
+    struct msg_tqh     imsg_q;        /* incoming request Q */
+    struct msg_tqh     omsg_q;        /* outstanding request Q */
+    struct msg         *rmsg;         /* current message being rcvd */
+    struct msg         *smsg;         /* current message being sent */
+
+    conn_recv_t        recv;          /* recv (read) handler */
+    conn_recv_next_t   recv_next;     /* recv next message handler */
+    conn_recv_done_t   recv_done;     /* read done handler */
+    conn_send_t        send;          /* send (write) handler */
+    conn_send_next_t   send_next;     /* write next message handler */
+    conn_send_done_t   send_done;     /* write done handler */
+    conn_close_t       close;         /* close handler */
+    conn_active_t      active;        /* active? handler */
+
+    conn_ref_t         ref;           /* connection reference handler */
+    conn_unref_t       unref;         /* connection unreference handler */
+
+    conn_msgq_t        enqueue_inq;   /* connection inq msg enqueue handler */
+    conn_msgq_t        dequeue_inq;   /* connection inq msg dequeue handler */
+    conn_msgq_t        enqueue_outq;  /* connection outq msg enqueue handler */
+    conn_msgq_t        dequeue_outq;  /* connection outq msg dequeue handler */
+
+    size_t             recv_bytes;    /* received (read) bytes */
+    size_t             send_bytes;    /* sent (written) bytes */
+
+    uint32_t           events;        /* connection io events */
+    err_t              err;           /* connection errno */
+    unsigned           recv_active:1; /* recv active? */
+    unsigned           recv_ready:1;  /* recv ready? */
+    unsigned           send_active:1; /* send active? */
+    unsigned           send_ready:1;  /* send ready? */
+
+    unsigned           client:1;      /* client? or server? */
+    unsigned           proxy:1;       /* proxy? */
+    unsigned           connecting:1;  /* connecting? */
+    unsigned           connected:1;   /* connected? */
+    unsigned           eof:1;         /* eof? aka passive close? */
+    unsigned           done:1;        /* done? aka close? */
+    unsigned           redis:1;       /* redis? */
+};
+
+TAILQ_HEAD(conn_tqh, conn);
+
+struct conn *conn_get(void *owner, bool client, bool redis);
+struct conn *conn_get_proxy(void *owner);
+void conn_put(struct conn *conn);
+ssize_t conn_recv(struct conn *conn, void *buf, size_t size);
+ssize_t conn_sendv(struct conn *conn, struct array *sendv, size_t nsend);
+void conn_init(void);
+void conn_deinit(void);
+
+#endif

data/ext/nutcracker/src/nc_core.c

@@ -0,0 +1,334 @@
+/*
+ * twemproxy - A fast and lightweight proxy for memcached protocol.
+ * Copyright (C) 2011 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdlib.h>
+#include <unistd.h>
+
+#include <sys/epoll.h>
+
+#include <nc_core.h>
+#include <nc_event.h>
+#include <nc_conf.h>
+#include <nc_server.h>
+#include <nc_proxy.h>
+
+static uint32_t ctx_id; /* context generation */
+
+static struct context *
+core_ctx_create(struct instance *nci)
+{
+    rstatus_t status;
+    struct context *ctx;
+
+    ctx = nc_alloc(sizeof(*ctx));
+    if (ctx == NULL) {
+        return NULL;
+    }
+    ctx->id = ++ctx_id;
+    ctx->cf = NULL;
+    ctx->stats = NULL;
+    array_null(&ctx->pool);
+    ctx->ep = -1;
+    ctx->nevent = EVENT_SIZE_HINT;
+    ctx->max_timeout = nci->stats_interval;
+    ctx->timeout = ctx->max_timeout;
+    ctx->event = NULL;
+
+    /* parse and create configuration */
+    ctx->cf = conf_create(nci->conf_filename);
+    if (ctx->cf == NULL) {
+        nc_free(ctx);
+        return NULL;
+    }
+
+    /* initialize server pool from configuration */
+    status = server_pool_init(&ctx->pool, &ctx->cf->pool, ctx);
+    if (status != NC_OK) {
+        conf_destroy(ctx->cf);
+        nc_free(ctx);
+        return NULL;
+    }
+
+    /* create stats per server pool */
+    ctx->stats = stats_create(nci->stats_port, nci->stats_addr, nci->stats_interval,
+                              nci->hostname, &ctx->pool);
+    if (ctx->stats == NULL) {
+        server_pool_deinit(&ctx->pool);
+        conf_destroy(ctx->cf);
+        nc_free(ctx);
+        return NULL;
+    }
+
+    /* initialize event handling for client, proxy and server */
+    status = event_init(ctx, EVENT_SIZE_HINT);
+    if (status != NC_OK) {
+        stats_destroy(ctx->stats);
+        server_pool_deinit(&ctx->pool);
+        conf_destroy(ctx->cf);
+        nc_free(ctx);
+        return NULL;
+    }
+
+    /* preconnect? servers in server pool */
+    status = server_pool_preconnect(ctx);
+    if (status != NC_OK) {
+        server_pool_disconnect(ctx);
+        event_deinit(ctx);
+        stats_destroy(ctx->stats);
+        server_pool_deinit(&ctx->pool);
+        conf_destroy(ctx->cf);
+        nc_free(ctx);
+        return NULL;
+    }
+
+    /* initialize proxy per server pool */
+    status = proxy_init(ctx);
+    if (status != NC_OK) {
+        server_pool_disconnect(ctx);
+        event_deinit(ctx);
+        stats_destroy(ctx->stats);
+        server_pool_deinit(&ctx->pool);
+        conf_destroy(ctx->cf);
+        nc_free(ctx);
+        return NULL;
+    }
+
+    log_debug(LOG_VVERB, "created ctx %p id %"PRIu32"", ctx, ctx->id);
+
+    return ctx;
+}
+
+static void
+core_ctx_destroy(struct context *ctx)
+{
+    log_debug(LOG_VVERB, "destroy ctx %p id %"PRIu32"", ctx, ctx->id);
+    proxy_deinit(ctx);
+    server_pool_disconnect(ctx);
+    event_deinit(ctx);
+    stats_destroy(ctx->stats);
+    server_pool_deinit(&ctx->pool);
+    conf_destroy(ctx->cf);
+    nc_free(ctx);
+}
+
+struct context *
+core_start(struct instance *nci)
+{
+    struct context *ctx;
+
+    mbuf_init(nci);
+    msg_init();
+    conn_init();
+
+    ctx = core_ctx_create(nci);
+    if (ctx != NULL) {
+        nci->ctx = ctx;
+        return ctx;
+    }
+
+    conn_deinit();
+    msg_deinit();
+    mbuf_deinit();
+
+    return NULL;
+}
+
+void
+core_stop(struct context *ctx)
+{
+    conn_deinit();
+    msg_deinit();
+    mbuf_deinit();
+    core_ctx_destroy(ctx);
+}
+
+static rstatus_t
+core_recv(struct context *ctx, struct conn *conn)
+{
+    rstatus_t status;
+
+    status = conn->recv(ctx, conn);
+    if (status != NC_OK) {
+        log_debug(LOG_INFO, "recv on %c %d failed: %s",
+                  conn->client ? 'c' : (conn->proxy ? 'p' : 's'), conn->sd,
+                  strerror(errno));
+    }
+
+    return status;
+}
+
+static rstatus_t
+core_send(struct context *ctx, struct conn *conn)
+{
+    rstatus_t status;
+
+    status = conn->send(ctx, conn);
+    if (status != NC_OK) {
+        log_debug(LOG_INFO, "send on %c %d failed: %s",
+                  conn->client ? 'c' : (conn->proxy ? 'p' : 's'), conn->sd,
+                  strerror(errno));
+    }
+
+    return status;
+}
+
+static void
+core_close(struct context *ctx, struct conn *conn)
+{
+    rstatus_t status;
+    char type, *addrstr;
+
+    ASSERT(conn->sd > 0);
+
+    if (conn->client) {
+        type = 'c';
+        addrstr = nc_unresolve_peer_desc(conn->sd);
+    } else {
+        type = conn->proxy ? 'p' : 's';
+        addrstr = nc_unresolve_addr(conn->addr, conn->addrlen);
+    }
+    log_debug(LOG_NOTICE, "close %c %d '%s' on event %04"PRIX32" eof %d done "
+              "%d rb %zu sb %zu%c %s", type, conn->sd, addrstr, conn->events,
+              conn->eof, conn->done, conn->recv_bytes, conn->send_bytes,
+              conn->err ? ':' : ' ', conn->err ? strerror(conn->err) : "");
+
+    status = event_del_conn(ctx->ep, conn);
+    if (status < 0) {
+        log_warn("event del conn e %d %c %d failed, ignored: %s", ctx->ep,
+                 type, conn->sd, strerror(errno));
+    }
+
+    conn->close(ctx, conn);
+}
+
+static void
+core_error(struct context *ctx, struct conn *conn)
+{
+    rstatus_t status;
+    char type = conn->client ? 'c' : (conn->proxy ? 'p' : 's');
+
+    status = nc_get_soerror(conn->sd);
+    if (status < 0) {
+        log_warn("get soerr on %c %d failed, ignored: %s", type, conn->sd,
+                  strerror(errno));
+    }
+    conn->err = errno;
+
+    core_close(ctx, conn);
+}
+
+static void
+core_timeout(struct context *ctx)
+{
+    for (;;) {
+        struct msg *msg;
+        struct conn *conn;
+        int64_t now, then;
+
+        msg = msg_tmo_min();
+        if (msg == NULL) {
+            ctx->timeout = ctx->max_timeout;
+            return;
+        }
+
+        /* skip over req that are in-error or done */
+
+        if (msg->error || msg->done) {
+            msg_tmo_delete(msg);
+            continue;
+        }
+
+        /*
+         * timeout expired req and all the outstanding req on the timing
+         * out server
+         */
+
+        conn = msg->tmo_rbe.data;
+        then = msg->tmo_rbe.key;
+
+        now = nc_msec_now();
+        if (now < then) {
+            int delta = (int)(then - now);
+            ctx->timeout = MIN(delta, ctx->max_timeout);
+            return;
+        }
+
+        log_debug(LOG_INFO, "req %"PRIu64" on s %d timedout", msg->id, conn->sd);
+
+        msg_tmo_delete(msg);
+        conn->err = ETIMEDOUT;
+
+        core_close(ctx, conn);
+    }
+}
+
+static void
+core_core(struct context *ctx, struct conn *conn, uint32_t events)
+{
+    rstatus_t status;
+
+    log_debug(LOG_VVERB, "event %04"PRIX32" on %c %d", events,
+              conn->client ? 'c' : (conn->proxy ? 'p' : 's'), conn->sd);
+
+    conn->events = events;
+
+    /* error takes precedence over read | write */
+    if (events & EPOLLERR) {
+        core_error(ctx, conn);
+        return;
+    }
+
+    /* read takes precedence over write */
+    if (events & (EPOLLIN | EPOLLHUP)) {
+        status = core_recv(ctx, conn);
+        if (status != NC_OK || conn->done || conn->err) {
+            core_close(ctx, conn);
+            return;
+        }
+    }
+
+    if (events & EPOLLOUT) {
+        status = core_send(ctx, conn);
+        if (status != NC_OK || conn->done || conn->err) {
+            core_close(ctx, conn);
+            return;
+        }
+    }
+}
+
+rstatus_t
+core_loop(struct context *ctx)
+{
+    int i, nsd;
+
+    nsd = event_wait(ctx->ep, ctx->event, ctx->nevent, ctx->timeout);
+    if (nsd < 0) {
+        return nsd;
+    }
+
+    for (i = 0; i < nsd; i++) {
+        struct epoll_event *ev = &ctx->event[i];
+
+        core_core(ctx, ev->data.ptr, ev->events);
+    }
+
+    core_timeout(ctx);
+
+    stats_swap(ctx->stats);
+
+    return NC_OK;
+}

data/ext/nutcracker/src/nc_core.h

@@ -0,0 +1,131 @@
+/*
+ * twemproxy - A fast and lightweight proxy for memcached protocol.
+ * Copyright (C) 2011 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _NC_CORE_H_
+#define _NC_CORE_H_
+
+#ifdef HAVE_CONFIG_H
+# include <config.h>
+#endif
+
+#ifdef HAVE_DEBUG_LOG
+# define NC_DEBUG_LOG 1
+#endif
+
+#ifdef HAVE_ASSERT_PANIC
+# define NC_ASSERT_PANIC 1
+#endif
+
+#ifdef HAVE_ASSERT_LOG
+# define NC_ASSERT_LOG 1
+#endif
+
+#ifdef HAVE_STATS
+# define NC_STATS 1
+#else
+# define NC_STATS 0
+#endif
+
+#ifdef HAVE_LITTLE_ENDIAN
+# define NC_LITTLE_ENDIAN 1
+#endif
+
+#define NC_OK        0
+#define NC_ERROR    -1
+#define NC_EAGAIN   -2
+#define NC_ENOMEM   -3
+
+typedef int rstatus_t; /* return type */
+typedef int err_t;     /* error type */
+
+struct array;
+struct string;
+struct context;
+struct conn;
+struct conn_tqh;
+struct msg;
+struct msg_tqh;
+struct server;
+struct server_pool;
+struct mbuf;
+struct mhdr;
+struct conf;
+struct stats;
+struct epoll_event;
+struct instance;
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <inttypes.h>
+#include <string.h>
+#include <errno.h>
+#include <limits.h>
+#include <time.h>
+#include <unistd.h>
+#include <pthread.h>
+
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <netinet/in.h>
+
+#include <nc_array.h>
+#include <nc_string.h>
+#include <nc_queue.h>
+#include <nc_rbtree.h>
+#include <nc_log.h>
+#include <nc_util.h>
+#include <nc_stats.h>
+#include <nc_mbuf.h>
+#include <nc_message.h>
+#include <nc_connection.h>
+
+struct context {
+    uint32_t           id;          /* unique context id */
+    struct conf        *cf;         /* configuration */
+    struct stats       *stats;      /* stats */
+
+    struct array       pool;        /* server_pool[] */
+
+    int                ep;          /* epoll device */
+    int                nevent;      /* # epoll event */
+    int                max_timeout; /* epoll wait max timeout in msec */
+    int                timeout;     /* epoll wait timeout in msec */
+    struct epoll_event *event;      /* epoll event */
+};
+
+struct instance {
+    struct context  *ctx;                        /* active context */
+    int             log_level;                   /* log level */
+    char            *log_filename;               /* log filename */
+    char            *conf_filename;              /* configuration filename */
+    uint16_t        stats_port;                  /* stats monitoring port */
+    int             stats_interval;              /* stats aggregation interval */
+    char            *stats_addr;                 /* stats monitoring addr */
+    char            hostname[NC_MAXHOSTNAMELEN]; /* hostname */
+    size_t          mbuf_chunk_size;             /* mbuf chunk size */
+    pid_t           pid;                         /* process id */
+    char            *pid_filename;               /* pid filename */
+    unsigned        pidfile:1;                   /* pid file created? */
+};
+
+struct context *core_start(struct instance *nci);
+void core_stop(struct context *ctx);
+rstatus_t core_loop(struct context *ctx);
+
+#endif