nutcracker 0.2.3

data/ext/nutcracker/src/nc_log.h

@@ -0,0 +1,120 @@
+/*
+ * twemproxy - A fast and lightweight proxy for memcached protocol.
+ * Copyright (C) 2011 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _NC_LOG_H_
+#define _NC_LOG_H_
+
+struct logger {
+    char *name;  /* log file name */
+    int  level;  /* log level */
+    int  fd;     /* log file descriptor */
+    int  nerror; /* # log error */
+};
+
+#define LOG_EMERG   0   /* system in unusable */
+#define LOG_ALERT   1   /* action must be taken immediately */
+#define LOG_CRIT    2   /* critical conditions */
+#define LOG_ERR     3   /* error conditions */
+#define LOG_WARN    4   /* warning conditions */
+#define LOG_NOTICE  5   /* normal but significant condition (default) */
+#define LOG_INFO    6   /* informational */
+#define LOG_DEBUG   7   /* debug messages */
+#define LOG_VERB    8   /* verbose messages */
+#define LOG_VVERB   9   /* verbose messages on crack */
+#define LOG_VVVERB  10  /* verbose messages on ganga */
+#define LOG_PVERB   11  /* periodic verbose messages on crack */
+
+#define LOG_MAX_LEN 256 /* max length of log message */
+
+/*
+ * log_stderr   - log to stderr
+ * loga         - log always
+ * loga_hexdump - log hexdump always
+ * log_error    - error log messages
+ * log_warn     - warning log messages
+ * log_panic    - log messages followed by a panic
+ * ...
+ * log_debug    - debug log messages based on a log level
+ * log_hexdump  - hexadump -C of a log buffer
+ */
+#ifdef NC_DEBUG_LOG
+
+#define log_debug(_level, ...) do {                                         \
+    if (log_loggable(_level) != 0) {                                        \
+        _log(__FILE__, __LINE__, 0, __VA_ARGS__);                           \
+    }                                                                       \
+} while (0)
+
+#define log_hexdump(_level, _data, _datalen, ...) do {                      \
+    if (log_loggable(_level) != 0) {                                        \
+        _log(__FILE__, __LINE__, 0, __VA_ARGS__);                           \
+        _log_hexdump(__FILE__, __LINE__, (char *)(_data), (int)(_datalen),  \
+                     __VA_ARGS__);                                          \
+    }                                                                       \
+} while (0)
+
+#else
+
+#define log_debug(_level, ...)
+#define log_hexdump(_level, _data, _datalen, ...)
+
+#endif
+
+#define log_stderr(...) do {                                                \
+    _log_stderr(__VA_ARGS__);                                               \
+} while (0)
+
+#define loga(...) do {                                                      \
+    _log(__FILE__, __LINE__, 0, __VA_ARGS__);                               \
+} while (0)
+
+#define loga_hexdump(_data, _datalen, ...) do {                             \
+    _log(__FILE__, __LINE__, 0, __VA_ARGS__);                               \
+    _log_hexdump(__FILE__, __LINE__, (char *)(_data), (int)(_datalen),      \
+                 __VA_ARGS__);                                              \
+} while (0)                                                                 \
+
+#define log_error(...) do {                                                 \
+    if (log_loggable(LOG_ALERT) != 0) {                                     \
+        _log(__FILE__, __LINE__, 0, __VA_ARGS__);                           \
+    }                                                                       \
+} while (0)
+
+#define log_warn(...) do {                                                  \
+    if (log_loggable(LOG_WARN) != 0) {                                      \
+        _log(__FILE__, __LINE__, 0, __VA_ARGS__);                           \
+    }                                                                       \
+} while (0)
+
+#define log_panic(...) do {                                                 \
+    if (log_loggable(LOG_EMERG) != 0) {                                     \
+        _log(__FILE__, __LINE__, 1, __VA_ARGS__);                           \
+    }                                                                       \
+} while (0)
+
+int log_init(int level, char *filename);
+void log_deinit(void);
+void log_level_up(void);
+void log_level_down(void);
+void log_level_set(int level);
+void log_reopen(void);
+int log_loggable(int level);
+void _log(const char *file, int line, int panic, const char *fmt, ...);
+void _log_stderr(const char *fmt, ...);
+void _log_hexdump(const char *file, int line, char *data, int datalen, const char *fmt, ...);
+
+#endif

data/ext/nutcracker/src/nc_mbuf.c

@@ -0,0 +1,285 @@
+/*
+ * twemproxy - A fast and lightweight proxy for memcached protocol.
+ * Copyright (C) 2011 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdlib.h>
+#include <string.h>
+
+#include <nc_core.h>
+
+static uint32_t nfree_mbufq;   /* # free mbuf */
+static struct mhdr free_mbufq; /* free mbuf q */
+
+static size_t mbuf_chunk_size; /* mbuf chunk size - header + data (const) */
+static size_t mbuf_offset;     /* mbuf offset in chunk (const) */
+
+static struct mbuf *
+_mbuf_get(void)
+{
+    struct mbuf *mbuf;
+    uint8_t *buf;
+
+    if (!STAILQ_EMPTY(&free_mbufq)) {
+        ASSERT(nfree_mbufq > 0);
+
+        mbuf = STAILQ_FIRST(&free_mbufq);
+        nfree_mbufq--;
+        STAILQ_REMOVE_HEAD(&free_mbufq, next);
+
+        ASSERT(mbuf->magic == MBUF_MAGIC);
+        goto done;
+    }
+
+    buf = nc_alloc(mbuf_chunk_size);
+    if (buf == NULL) {
+        return NULL;
+    }
+
+    /*
+     * mbuf header is at the tail end of the mbuf. This enables us to catch
+     * buffer overrun early by asserting on the magic value during get or
+     * put operations
+     *
+     *   <------------- mbuf_chunk_size ------------->
+     *   +-------------------------------------------+
+     *   |       mbuf data          |  mbuf header   |
+     *   |     (mbuf_offset)        | (struct mbuf)  |
+     *   +-------------------------------------------+
+     *   ^           ^        ^     ^^
+     *   |           |        |     ||
+     *   \           |        |     |\
+     *   mbuf->start \        |     | mbuf->end (one byte past valid bound)
+     *                mbuf->pos     \
+     *                        \      mbuf
+     *                        mbuf->last (one byte past valid byte)
+     *
+     */
+    mbuf = (struct mbuf *)(buf + mbuf_offset);
+    mbuf->magic = MBUF_MAGIC;
+
+done:
+    STAILQ_NEXT(mbuf, next) = NULL;
+    return mbuf;
+}
+
+struct mbuf *
+mbuf_get(void)
+{
+    struct mbuf *mbuf;
+    uint8_t *buf;
+
+    mbuf = _mbuf_get();
+    if (mbuf == NULL) {
+        return NULL;
+    }
+
+    buf = (uint8_t *)mbuf - mbuf_offset;
+    mbuf->start = buf;
+    mbuf->end = buf + mbuf_offset;
+
+    ASSERT(mbuf->end - mbuf->start == (int)mbuf_offset);
+    ASSERT(mbuf->start < mbuf->end);
+
+    mbuf->pos = mbuf->start;
+    mbuf->last = mbuf->start;
+
+    log_debug(LOG_VVERB, "get mbuf %p", mbuf);
+
+    return mbuf;
+}
+
+static void
+mbuf_free(struct mbuf *mbuf)
+{
+    uint8_t *buf;
+
+    log_debug(LOG_VVERB, "put mbuf %p len %d", mbuf, mbuf->last - mbuf->pos);
+
+    ASSERT(STAILQ_NEXT(mbuf, next) == NULL);
+    ASSERT(mbuf->magic == MBUF_MAGIC);
+
+    buf = (uint8_t *)mbuf - mbuf_offset;
+    nc_free(buf);
+}
+
+void
+mbuf_put(struct mbuf *mbuf)
+{
+    log_debug(LOG_VVERB, "put mbuf %p len %d", mbuf, mbuf->last - mbuf->pos);
+
+    ASSERT(STAILQ_NEXT(mbuf, next) == NULL);
+    ASSERT(mbuf->magic == MBUF_MAGIC);
+
+    nfree_mbufq++;
+    STAILQ_INSERT_HEAD(&free_mbufq, mbuf, next);
+}
+
+/*
+ * Rewind the mbuf by discarding any of the read or unread data that it
+ * might hold.
+ */
+void
+mbuf_rewind(struct mbuf *mbuf)
+{
+    mbuf->pos = mbuf->start;
+    mbuf->last = mbuf->start;
+}
+
+/*
+ * Return the length of data in mbuf. Mbuf cannot contain more than
+ * 2^32 bytes (4G).
+ */
+uint32_t
+mbuf_length(struct mbuf *mbuf)
+{
+    ASSERT(mbuf->last >= mbuf->pos);
+
+    return (uint32_t)(mbuf->last - mbuf->pos);
+}
+
+/*
+ * Return the remaining space size for any new data in mbuf. Mbuf cannot
+ * contain more than 2^32 bytes (4G).
+ */
+uint32_t
+mbuf_size(struct mbuf *mbuf)
+{
+    ASSERT(mbuf->end >= mbuf->last);
+
+    return (uint32_t)(mbuf->end - mbuf->last);
+}
+
+/*
+ * Return the maximum available space size for data in any mbuf. Mbuf cannot
+ * contain more than 2^32 bytes (4G).
+ */
+size_t
+mbuf_data_size(void)
+{
+    return mbuf_offset;
+}
+
+/*
+ * Insert mbuf at the tail of the mhdr Q
+ */
+void
+mbuf_insert(struct mhdr *mhdr, struct mbuf *mbuf)
+{
+    STAILQ_INSERT_TAIL(mhdr, mbuf, next);
+    log_debug(LOG_VVERB, "insert mbuf %p len %d", mbuf, mbuf->last - mbuf->pos);
+}
+
+/*
+ * Remove mbuf from the mhdr Q
+ */
+void
+mbuf_remove(struct mhdr *mhdr, struct mbuf *mbuf)
+{
+    log_debug(LOG_VVERB, "remove mbuf %p len %d", mbuf, mbuf->last - mbuf->pos);
+
+    STAILQ_REMOVE(mhdr, mbuf, mbuf, next);
+    STAILQ_NEXT(mbuf, next) = NULL;
+}
+
+/*
+ * Copy n bytes from memory area pos to mbuf.
+ *
+ * The memory areas should not overlap and the mbuf should have
+ * enough space for n bytes.
+ */
+void
+mbuf_copy(struct mbuf *mbuf, uint8_t *pos, size_t n)
+{
+    if (n == 0) {
+        return;
+    }
+
+    /* mbuf has space for n bytes */
+    ASSERT(!mbuf_full(mbuf) && n <= mbuf_size(mbuf));
+
+    /* no overlapping copy */
+    ASSERT(pos < mbuf->start || pos >= mbuf->end);
+
+    nc_memcpy(mbuf->last, pos, n);
+    mbuf->last += n;
+}
+
+/*
+ * Split mbuf h into h and t by copying data from h to t. Before
+ * the copy, we invoke a precopy handler cb that will copy a predefined
+ * string to the head of t.
+ *
+ * Return new mbuf t, if the split was successful.
+ */
+struct mbuf *
+mbuf_split(struct mhdr *h, uint8_t *pos, mbuf_copy_t cb, void *cbarg)
+{
+    struct mbuf *mbuf, *nbuf;
+    size_t size;
+
+    ASSERT(!STAILQ_EMPTY(h));
+
+    mbuf = STAILQ_LAST(h, mbuf, next);
+    ASSERT(pos >= mbuf->pos && pos <= mbuf->last);
+
+    nbuf = mbuf_get();
+    if (nbuf == NULL) {
+        return NULL;
+    }
+
+    if (cb != NULL) {
+        /* precopy nbuf */
+        cb(nbuf, cbarg);
+    }
+
+    /* copy data from mbuf to nbuf */
+    size = (size_t)(mbuf->last - pos);
+    mbuf_copy(nbuf, pos, size);
+
+    /* adjust mbuf */
+    mbuf->last = pos;
+
+    log_debug(LOG_VVERB, "split into mbuf %p len %"PRIu32" and nbuf %p len "
+              "%"PRIu32" copied %zu bytes", mbuf, mbuf_length(mbuf), nbuf,
+              mbuf_length(nbuf), size);
+
+    return nbuf;
+}
+
+void
+mbuf_init(struct instance *nci)
+{
+    nfree_mbufq = 0;
+    STAILQ_INIT(&free_mbufq);
+
+    mbuf_chunk_size = nci->mbuf_chunk_size;
+    mbuf_offset = mbuf_chunk_size - MBUF_HSIZE;
+
+    log_debug(LOG_DEBUG, "mbuf hsize %d chunk size %zu offset %zu length %zu",
+              MBUF_HSIZE, mbuf_chunk_size, mbuf_offset, mbuf_offset);
+}
+
+void
+mbuf_deinit(void)
+{
+    while (!STAILQ_EMPTY(&free_mbufq)) {
+        struct mbuf *mbuf = STAILQ_FIRST(&free_mbufq);
+        mbuf_remove(&free_mbufq, mbuf);
+        mbuf_free(mbuf);
+        nfree_mbufq--;
+    }
+    ASSERT(nfree_mbufq == 0);
+}

data/ext/nutcracker/src/nc_mbuf.h

@@ -0,0 +1,67 @@
+/*
+ * twemproxy - A fast and lightweight proxy for memcached protocol.
+ * Copyright (C) 2011 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _NC_MBUF_H_
+#define _NC_MBUF_H_
+
+#include <nc_core.h>
+
+typedef void (*mbuf_copy_t)(struct mbuf *, void *);
+
+struct mbuf {
+    uint32_t           magic;   /* mbuf magic (const) */
+    STAILQ_ENTRY(mbuf) next;    /* next mbuf */
+    uint8_t            *pos;    /* read marker */
+    uint8_t            *last;   /* write marker */
+    uint8_t            *start;  /* start of buffer (const) */
+    uint8_t            *end;    /* end of buffer (const) */
+};
+
+STAILQ_HEAD(mhdr, mbuf);
+
+#define MBUF_MAGIC      0xdeadbeef
+#define MBUF_MIN_SIZE   512
+#define MBUF_MAX_SIZE   65536
+#define MBUF_SIZE       16384
+#define MBUF_HSIZE      sizeof(struct mbuf)
+
+static inline bool
+mbuf_empty(struct mbuf *mbuf)
+{
+    return mbuf->pos == mbuf->last ? true : false;
+}
+
+static inline bool
+mbuf_full(struct mbuf *mbuf)
+{
+    return mbuf->last == mbuf->end ? true : false;
+}
+
+void mbuf_init(struct instance *nci);
+void mbuf_deinit(void);
+struct mbuf *mbuf_get(void);
+void mbuf_put(struct mbuf *mbuf);
+void mbuf_rewind(struct mbuf *mbuf);
+uint32_t mbuf_length(struct mbuf *mbuf);
+uint32_t mbuf_size(struct mbuf *mbuf);
+size_t mbuf_data_size(void);
+void mbuf_insert(struct mhdr *mhdr, struct mbuf *mbuf);
+void mbuf_remove(struct mhdr *mhdr, struct mbuf *mbuf);
+void mbuf_copy(struct mbuf *mbuf, uint8_t *pos, size_t n);
+struct mbuf *mbuf_split(struct mhdr *h, uint8_t *pos, mbuf_copy_t cb, void *cbarg);
+
+#endif