nutcracker 0.2.4.4 → 0.2.4.5

data/ext/nutcracker/src/nc_core.h

@@ -40,10 +40,20 @@
 # define NC_STATS 0
 #endif
 
+#ifdef HAVE_EPOLL
+#define NC_HAVE_EPOLL 1
+#elif HAVE_KQUEUE
+#define NC_HAVE_KQUEUE 1
+#endif
+
 #ifdef HAVE_LITTLE_ENDIAN
 # define NC_LITTLE_ENDIAN 1
 #endif
 
+#ifdef HAVE_BACKTRACE
+#define NC_HAVE_BACKTRACE 1
+#endif
+
 #define NC_OK        0
 #define NC_ERROR    -1
 #define NC_EAGAIN   -2
@@ -101,14 +111,12 @@ struct context {
     struct stats       *stats;      /* stats */
 
     struct array       pool;        /* server_pool[] */
-
-    int                ep;          /* epoll device */
-    int                nevent;      /* # epoll event */
+    struct evbase      *evb;
     int                max_timeout; /* epoll wait max timeout in msec */
-    int                timeout;     /* epoll wait timeout in msec */
-    struct epoll_event *event;      /* epoll event */
+    int                timeout;
 };
 
+
 struct instance {
     struct context  *ctx;                        /* active context */
     int             log_level;                   /* log level */

data/ext/nutcracker/src/{nc_event.c → nc_epoll.c}

@@ -16,66 +16,86 @@
  */
 
 #include <unistd.h>
-#include <sys/epoll.h>
-
 #include <nc_core.h>
 #include <nc_event.h>
 
-int
-event_init(struct context *ctx, int size)
+#ifdef NC_HAVE_EPOLL
+#include <sys/epoll.h>
+
+struct evbase * 
+evbase_create(int nevent, void (*callback_fp)(void *, uint32_t))
 {
+
+    struct evbase *evb;
     int status, ep;
     struct epoll_event *event;
 
-    ASSERT(ctx->ep < 0);
-    ASSERT(ctx->nevent != 0);
-    ASSERT(ctx->event == NULL);
+    if (nevent <= 0) {
+        log_error("nevent has to be positive %d", nevent);
+        return NULL;
+    }
 
-    ep = epoll_create(size);
+    ep = epoll_create(nevent);
     if (ep < 0) {
-        log_error("epoll create of size %d failed: %s", size, strerror(errno));
-        return -1;
+        log_error("epoll create of size %d failed: %s", nevent, strerror(errno));
+        return NULL;
     }
 
-    event = nc_calloc(ctx->nevent, sizeof(*ctx->event));
+    event = nc_calloc(nevent, sizeof(*event));
     if (event == NULL) {
         status = close(ep);
         if (status < 0) {
             log_error("close e %d failed, ignored: %s", ep, strerror(errno));
         }
-        return -1;
+        return NULL;
     }
 
-    ctx->ep = ep;
-    ctx->event = event;
+    evb = nc_alloc(sizeof(*evb));
+    if (evb == NULL) {
+        nc_free(event);
+        status = close(ep);
+        if (status < 0) {
+            log_error("close e %d failed, ignored: %s", ep, strerror(errno));
+        }
+        return NULL;
 
-    log_debug(LOG_INFO, "e %d with nevent %d timeout %d", ctx->ep,
-              ctx->nevent, ctx->timeout);
+    }
 
-    return 0;
+    evb->nevent = nevent;
+    evb->ep = ep;
+    evb->event = event;
+    evb->callback_fp = callback_fp;
+
+    log_debug(LOG_INFO, "e %d with nevent %d", evb->ep,
+              evb->nevent);
+
+    return evb;
 }
 
 void
-event_deinit(struct context *ctx)
+evbase_destroy(struct evbase *evb)
 {
     int status;
 
-    ASSERT(ctx->ep >= 0);
+    if (evb == NULL) return;
 
-    nc_free(ctx->event);
+    ASSERT(evb->ep >= 0);
 
-    status = close(ctx->ep);
+    nc_free(evb->event);
+
+    status = close(evb->ep);
     if (status < 0) {
-        log_error("close e %d failed, ignored: %s", ctx->ep, strerror(errno));
+        log_error("close e %d failed, ignored: %s", evb->ep, strerror(errno));
     }
-    ctx->ep = -1;
+    nc_free(evb);
 }
 
 int
-event_add_out(int ep, struct conn *c)
+event_add_out(struct evbase *evb, struct conn *c)
 {
     int status;
     struct epoll_event event;
+    int ep = evb->ep;
 
     ASSERT(ep > 0);
     ASSERT(c != NULL);
@@ -101,10 +121,11 @@ event_add_out(int ep, struct conn *c)
 }
 
 int
-event_del_out(int ep, struct conn *c)
+event_del_out(struct evbase *evb, struct conn *c)
 {
     int status;
     struct epoll_event event;
+    int ep = evb->ep;
 
     ASSERT(ep > 0);
     ASSERT(c != NULL);
@@ -130,10 +151,11 @@ event_del_out(int ep, struct conn *c)
 }
 
 int
-event_add_conn(int ep, struct conn *c)
+event_add_conn(struct evbase *evb, struct conn *c)
 {
     int status;
     struct epoll_event event;
+    int ep = evb->ep;
 
     ASSERT(ep > 0);
     ASSERT(c != NULL);
@@ -155,9 +177,10 @@ event_add_conn(int ep, struct conn *c)
 }
 
 int
-event_del_conn(int ep, struct conn *c)
+event_del_conn(struct evbase *evb, struct conn *c)
 {
     int status;
+    int ep = evb->ep;
 
     ASSERT(ep > 0);
     ASSERT(c != NULL);
@@ -176,9 +199,14 @@ event_del_conn(int ep, struct conn *c)
 }
 
 int
-event_wait(int ep, struct epoll_event *event, int nevent, int timeout)
+event_wait(struct evbase *evb, int timeout)
 {
-    int nsd;
+    int nsd, i;
+    uint32_t evflags = 0;
+    int ep = evb->ep;
+    struct epoll_event *event = evb->event;
+    int nevent = evb->nevent;
+    void (*callback_fp)(void *, uint32_t) = evb->callback_fp;
 
     ASSERT(ep > 0);
     ASSERT(event != NULL);
@@ -187,6 +215,22 @@ event_wait(int ep, struct epoll_event *event, int nevent, int timeout)
     for (;;) {
         nsd = epoll_wait(ep, event, nevent, timeout);
         if (nsd > 0) {
+            for (i = 0; i < nsd; i++) {
+                struct epoll_event *ev = &evb->event[i];
+
+                evflags = 0;
+                if (ev->events & EPOLLERR)
+                    evflags |= EV_ERR;
+
+                if (ev->events & EPOLLIN)
+                    evflags |= EV_READ;
+
+                if (ev->events & EPOLLOUT)
+                    evflags |= EV_WRITE;
+
+                if (callback_fp != NULL)
+                    (*callback_fp)((void *) ev->data.ptr, evflags);
+            }
             return nsd;
         }
 
@@ -209,6 +253,26 @@ event_wait(int ep, struct epoll_event *event, int nevent, int timeout)
 
         return -1;
     }
-
     NOT_REACHED();
 }
+
+int
+event_add_st(struct evbase *evb, int fd)
+{
+    int status;
+    struct epoll_event ev;
+
+    ev.data.fd = fd;
+    ev.events = EPOLLIN;
+
+    status = epoll_ctl(evb->ep, EPOLL_CTL_ADD, fd, &ev);
+    if (status < 0) {
+        log_error("epoll ctl on e %d sd %d failed: %s", evb->ep, fd,
+                  strerror(errno));
+        return status;
+    }
+    
+    return status;
+}
+
+#endif /* NC_HAVE_EPOLL */

data/ext/nutcracker/src/nc_event.h

@@ -20,20 +20,40 @@
 
 #include <nc_core.h>
 
-/*
- * A hint to the kernel that is used to size the event backing store
- * of a given epoll instance
- */
-#define EVENT_SIZE_HINT 1024
+#define NC_EVENT_SIZE 1024
+#define EV_READ 0xff
+#define EV_WRITE 0xff00
+#define EV_ERR 0xff0000
 
-int event_init(struct context *ctx, int size);
-void event_deinit(struct context *ctx);
+#ifdef NC_HAVE_KQUEUE
+struct evbase {
+    int                  kq;
+    struct kevent        *changes;   /* list of changes to be made */
+    struct kevent        *kevents;   /* list of events returned from kevent */
+    int                  n_changes;  /* number of changes in our list */
+    int                  n_returned; /* number of events returned from kevent */
+    int                  n_processed;
+    int                  nevent;
+    void (*callback_fp)(void *, uint32_t);
+};
+#endif
+#ifdef NC_HAVE_EPOLL
+struct evbase {
+    int                   ep;
+    int                   nevent;
+    struct epoll_event    *event;
+    void (*callback_fp)(void *, uint32_t);
+};
+#endif
 
-int event_add_out(int ep, struct conn *c);
-int event_del_out(int ep, struct conn *c);
-int event_add_conn(int ep, struct conn *c);
-int event_del_conn(int ep, struct conn *c);
+struct evbase *evbase_create(int size, void (*callback_fp)(void *, uint32_t));
+void evbase_destroy(struct evbase *evb);
 
-int event_wait(int ep, struct epoll_event *event, int nevent, int timeout);
+int event_add_out(struct evbase *evb, struct conn *c);
+int event_del_out(struct evbase *evb, struct conn *c);
+int event_add_conn(struct evbase *evb, struct conn *c);
+int event_del_conn(struct evbase *evb, struct conn *c);
+int event_wait(struct evbase *evb, int timeout);
+int event_add_st(struct evbase *evb, int fd);
 
-#endif
+#endif /* _NC_EVENT_H */

data/ext/nutcracker/src/nc_kqueue.c

@@ -0,0 +1,296 @@
+/*
+ * twemproxy - A fast and lightweight proxy for memcached protocol.
+ * Copyright (C) 2011 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+#include <unistd.h>
+#include <nc_core.h>
+#include <nc_event.h>
+
+#ifdef NC_HAVE_KQUEUE
+#include <sys/event.h>
+
+struct evbase *
+evbase_create(int nevent, void (*callback_fp)(void *, uint32_t))
+{
+
+    struct evbase *evb;
+    int status, kq;
+    struct kevent *changes, *kevents;
+
+    if (nevent <= 0) {
+        log_error("nevent has to be positive %d", nevent);
+        return NULL;
+    }
+
+    /* Initialize the kernel queue */
+    if ((kq = kqueue()) == -1) {
+        log_error("kernel queue create failed: %s", kq, strerror(errno));
+        return NULL;
+    }
+
+    changes = nc_calloc(nevent, sizeof(*changes));
+    if (changes == NULL) {
+        status = close(kq);
+        if (status < 0) {
+            log_error("close kq %d failed, ignored: %s", kq, strerror(errno));
+        }
+        return NULL;
+    }
+
+    kevents = nc_calloc(nevent, sizeof(*kevents));
+    if (kevents == NULL) {
+        nc_free(changes);
+        status = close(kq);
+        if (status < 0) {
+            log_error("close kq %d failed, ignored: %s", kq, strerror(errno));
+        }
+        return NULL;
+    }
+
+    evb = (struct evbase *) nc_alloc(sizeof(*evb));
+    if (evb == NULL) {
+        nc_free(changes);
+        nc_free(kevents);
+        status = close(kq);
+        if (status < 0) {
+            log_error("close kq %d failed, ignored: %s", kq, strerror(errno));
+        }
+        return NULL;
+    }
+
+    evb->kq = kq;
+    evb->changes = changes;
+    evb->kevents  = kevents;
+    evb->nevent = nevent; 
+    evb->callback_fp = callback_fp;
+    evb->n_changes = 0;
+
+    log_debug(LOG_INFO, "kq %d with nevent %d", evb->kq,
+              evb->nevent);
+
+    return evb;
+}
+
+void
+evbase_destroy(struct evbase *evb)
+{
+    int status;
+
+    if (evb == NULL) return;
+
+    ASSERT(evb->kq >= 0);
+
+    nc_free(evb->changes);
+    nc_free(evb->kevents);
+
+    status = close(evb->kq);
+    if (status < 0) {
+        log_error("close kq %d failed, ignored: %s", evb->kq, strerror(errno));
+    }
+    nc_free(evb);
+}
+
+int
+event_add_out(struct evbase *evb, struct conn *c)
+{
+    struct kevent *event;
+    int kq = evb->kq;
+
+    ASSERT(kq > 0);
+    ASSERT(c != NULL);
+    ASSERT(c->sd > 0);
+    ASSERT(c->recv_active);
+    ASSERT(evb->n_changes < evb->nevent);
+
+    if (c->send_active) {
+        return 0;
+    }
+
+    event = &evb->changes[(evb->n_changes)++];
+    EV_SET(event, c->sd, EVFILT_WRITE, EV_ADD | EV_CLEAR, 0, 0, (void *)c);
+
+    c->send_active = 1;
+
+    return 0;
+}
+
+int
+event_del_out(struct evbase *evb, struct conn *c)
+{
+    struct kevent *event;
+    int kq = evb->kq;
+
+    ASSERT(kq > 0);
+    ASSERT(c != NULL);
+    ASSERT(c->sd > 0);
+    ASSERT(c->recv_active);
+    ASSERT(evb->n_changes < evb->nevent);
+
+    if (!c->send_active) {
+        return 0;
+    }
+    
+    event = &evb->changes[(evb->n_changes)++];
+    EV_SET(event, c->sd, EVFILT_WRITE, EV_DELETE, 0, 0, (void *)c);
+
+    c->send_active = 0;
+
+    return 0;
+}
+
+int
+event_add_conn(struct evbase *evb, struct conn *c)
+{
+    struct kevent *event;
+    int kq = evb->kq;
+
+    ASSERT(kq > 0);
+    ASSERT(c != NULL);
+    ASSERT(c->sd > 0);
+    ASSERT(evb->n_changes < evb->nevent);
+
+    event = &evb->changes[(evb->n_changes)++];
+    EV_SET(event, c->sd, EVFILT_READ, EV_ADD | EV_CLEAR, 0, 0, (void *)c);
+
+    c->recv_active = 1;
+
+    event_add_out(evb, c);
+    c->send_active = 1;
+
+    return 0;
+}
+
+int
+event_del_conn(struct evbase *evb, struct conn *c)
+{
+    int i;
+    struct kevent *event;
+    int kq = evb->kq;
+
+    ASSERT(kq > 0);
+    ASSERT(c != NULL);
+    ASSERT(c->sd > 0);
+    ASSERT(evb->n_changes < evb->nevent);
+
+    event = &evb->changes[(evb->n_changes)++];
+    EV_SET(event, c->sd, EVFILT_READ, EV_DELETE, 0, 0, (void *)c);
+
+    event_del_out(evb, c);
+
+    c->recv_active = 0;
+    c->send_active = 0;
+    
+    /*
+     * Now, eliminate pending events for c->sd (there should be at most one
+     * other event). This is important because we will close c->sd and free
+     * c when we return.
+     */
+    for (i = evb->n_processed + 1; i < evb->n_returned; i++) {
+        struct kevent *ev = &evb->kevents[i];
+        if (ev->ident == (uintptr_t)c->sd) {
+            ev->flags = 0;
+            ev->filter = 0;
+            break;
+        }
+    }
+
+    return 0;
+}
+
+int
+event_wait(struct evbase *evb, int timeout)
+{
+    int kq = evb->kq;
+    struct timespec ts = nc_millisec_to_timespec(timeout);
+    void (*callback_fp)(void *, uint32_t) = evb->callback_fp;
+
+    ASSERT(kq > 0);
+
+    for (;;) {
+        evb->n_returned = kevent(kq, evb->changes, evb->n_changes, evb->kevents,
+                                 evb->nevent, &ts);
+        evb->n_changes = 0;
+        if (evb->n_returned > 0) {
+            for (evb->n_processed = 0; evb->n_processed < evb->n_returned;
+                 evb->n_processed++) {
+                struct kevent *ev = &evb->kevents[evb->n_processed];
+
+                uint32_t evflags = 0;
+                if (ev->flags & EV_ERROR) {
+                   /*
+                    * Error messages that can happen, when a delete fails.
+                    *   EBADF happens when the file descriptor has been
+                    *   closed,
+                    *   ENOENT when the file descriptor was closed and       
+                    *   then reopened.
+                    *   EINVAL for some reasons not understood; EINVAL
+                    *   should not be returned ever; but FreeBSD does :-\
+                    * An error is also indicated when a callback deletes
+                    * an event we are still processing.  In that case
+                    * the data field is set to ENOENT.
+                    */
+                    if (ev->data == EBADF ||
+                        ev->data == EINVAL ||
+                        ev->data == ENOENT)
+                        continue;
+                    evflags |= EV_ERR;
+                }
+
+                if (ev->filter == EVFILT_READ)
+                    evflags |= EV_READ;
+
+                if (ev->filter == EVFILT_WRITE)
+                    evflags |= EV_WRITE;
+
+                if (callback_fp != NULL && evflags != 0)
+                    (*callback_fp)((void *)(ev->udata), evflags);
+            }
+            return evb->n_returned;
+        }
+
+        if (evb->n_returned == 0) {
+            if (timeout == -1) {
+               log_error("kqueue on kq %d with %d events and %d timeout "
+                         "returned no events", kq, evb->nevent, timeout);
+                return -1;
+            }
+
+            return 0;
+        }
+
+        if (errno == EINTR) {
+            continue;
+        }
+
+        log_error("kevent on kq %d with %d events failed: %s", kq, evb->nevent,
+                  strerror(errno));
+
+        return -1;
+    }
+    NOT_REACHED();
+}
+
+int
+event_add_st(struct evbase *evb, int fd)
+{
+    struct kevent *ev = &evb->changes[(evb->n_changes)++]; 
+    EV_SET(ev, fd, EVFILT_READ, EV_ADD | EV_CLEAR, 0, 0, NULL);
+
+    return 0;
+}
+
+#endif /* NC_HAVE_KQUEUE */