mt-libcouchbase 1.4.0

data/ext/libcouchbase/src/logging.c

@@ -0,0 +1,244 @@
+/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *     Copyright 2014 Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#include "settings.h"
+#include "logging.h"
+#include "internal.h" /* for lcb_getenv* */
+#include <stdio.h>
+#include <stdarg.h>
+
+#ifdef _WIN32
+#define flockfile(x) (void) 0
+#define funlockfile(x) (void) 0
+#endif
+
+
+#if defined(unix) || defined(__unix__) || defined(__unix) || defined(_POSIX_VERSION)
+    #include <unistd.h>
+    #include <pthread.h>
+    #include <sys/types.h>
+
+    /** XXX: If any of these blocks give problems for your platform, just
+     * erase it and have it use the fallback implementation. This isn't core
+     * functionality of the library, but is a rather helpful feature in order
+     * to get the thread/process identifier
+     */
+
+    #if defined(__linux__)
+        #include <sys/syscall.h>
+        #define GET_THREAD_ID() (long)syscall(SYS_gettid)
+        #define THREAD_ID_FMT "ld"
+    #elif defined(__APPLE__)
+        #define GET_THREAD_ID() getpid(), pthread_mach_thread_np(pthread_self())
+        #define THREAD_ID_FMT "d/%x"
+    #elif defined(__sun) && defined(__SVR4)
+        #include <thread.h>
+        /* Thread IDs are not global in solaris, so it's nice to print the PID alongside it */
+        #define GET_THREAD_ID() getpid(), thr_self()
+        #define THREAD_ID_FMT "ld/%u"
+    #elif defined(__FreeBSD__)
+        /* Like solaris, but thr_self is a bit different here */
+        #include <sys/thr.h>
+        static long ret_thr_self(void) { long tmp; thr_self(&tmp); return tmp; }
+        #define GET_THREAD_ID() getpid(), ret_thr_self()
+        #define THREAD_ID_FMT "d/%ld"
+    #else
+        /* other unix? */
+        #define GET_THREAD_ID() 0
+        #define THREAD_ID_FMT "d"
+    #endif
+#elif defined(_WIN32)
+    #define GET_THREAD_ID() GetCurrentThreadId()
+    #define THREAD_ID_FMT "d"
+#else
+    #define GET_THREAD_ID() 0
+    #define THREAD_ID_FMT "d"
+#endif
+
+static hrtime_t start_time = 0;
+
+static void console_log(struct lcb_logprocs_st *procs,
+                        unsigned int iid,
+                        const char *subsys,
+                        int severity,
+                        const char *srcfile,
+                        int srcline,
+                        const char *fmt,
+                        va_list ap);
+
+static struct lcb_CONSOLELOGGER console_logprocs = {
+        {0 /* version */, {{console_log} /* v1 */} /*v*/},
+        NULL,
+        /** Minimum severity */
+        LCB_LOG_INFO
+};
+
+struct lcb_logprocs_st *lcb_console_logprocs = &console_logprocs.base;
+
+
+/**
+ * Return a string representation of the severity level
+ */
+static const char * level_to_string(int severity)
+{
+    switch (severity) {
+    case LCB_LOG_TRACE:
+        return "TRACE";
+    case LCB_LOG_DEBUG:
+        return "DEBUG";
+    case LCB_LOG_INFO:
+        return "INFO";
+    case LCB_LOG_WARN:
+        return "WARN";
+    case LCB_LOG_ERROR:
+        return "ERROR";
+    case LCB_LOG_FATAL:
+        return "FATAL";
+    default:
+        return "";
+    }
+}
+
+/**
+ * Default logging callback for the verbose logger.
+ */
+static void console_log(struct lcb_logprocs_st *procs,
+                        unsigned int iid,
+                        const char *subsys,
+                        int severity,
+                        const char *srcfile,
+                        int srcline,
+                        const char *fmt,
+                        va_list ap)
+{
+    FILE *fp;
+    hrtime_t now;
+    struct lcb_CONSOLELOGGER *vprocs = (struct lcb_CONSOLELOGGER *)procs;
+
+    if (severity < vprocs->minlevel) {
+        return;
+    }
+
+    if (!start_time) {
+        start_time = gethrtime();
+    }
+
+    now = gethrtime();
+    if (now == start_time) {
+        now++;
+    }
+
+    fp = vprocs->fp ? vprocs->fp : stderr;
+
+    flockfile(fp);
+    fprintf(fp, "%lums ", (unsigned long)(now - start_time) / 1000000);
+
+    fprintf(fp, "[I%08x] {%"THREAD_ID_FMT"} [%s] (%s - L:%d) ",
+            iid,
+            GET_THREAD_ID(),
+            level_to_string(severity),
+            subsys,
+            srcline);
+    vfprintf(fp, fmt, ap);
+    fprintf(fp, "\n");
+    funlockfile(fp);
+
+    (void)procs;
+    (void)srcfile;
+}
+
+
+LCB_INTERNAL_API
+void lcb_log(const struct lcb_settings_st *settings,
+             const char *subsys,
+             int severity,
+             const char *srcfile,
+             int srcline,
+             const char *fmt,
+             ...)
+{
+    va_list ap;
+    lcb_logging_callback callback;
+
+    if (!settings->logger) {
+        return;
+    }
+
+    if (settings->logger->version != 0) {
+        return;
+    }
+
+    callback = settings->logger->v.v0.callback;
+
+    va_start(ap, fmt);
+    callback(settings->logger, settings->iid, subsys, severity, srcfile, srcline, fmt, ap);
+    va_end(ap);
+}
+
+LCB_INTERNAL_API
+void lcb_log_badconfig(const struct lcb_settings_st *settings,
+    const char *subsys, int severity, const char *srcfile, int srcline,
+    const lcbvb_CONFIG *vbc, const char *origin_txt)
+{
+    const char *errstr = lcbvb_get_error(vbc);
+    if (!errstr) {
+        errstr = "<FIXME: No error string provided for parse failure>";
+    }
+
+    lcb_log(settings, subsys, severity, srcfile, srcline,
+        "vBucket config parsing failed: %s. Raw text in DEBUG level", errstr);
+    if (!origin_txt) {
+        origin_txt = "<FIXME: No origin text available>";
+    }
+    lcb_log(settings, subsys, LCB_LOG_DEBUG, srcfile, srcline, "%s", origin_txt);
+}
+
+lcb_logprocs * lcb_init_console_logger(void)
+{
+    char vbuf[1024];
+    char namebuf[PATH_MAX] = { 0 };
+    int lvl = 0;
+    int has_file = 0;
+
+    has_file = lcb_getenv_nonempty("LCB_LOGFILE", namebuf, sizeof(namebuf));
+    if (has_file && console_logprocs.fp == NULL) {
+        FILE *fp = fopen(namebuf, "a");
+        if (!fp) {
+            fprintf(stderr, "libcouchbase: could not open file '%s' for logging output. (%s)\n",
+                namebuf, strerror(errno));
+        }
+        console_logprocs.fp = fp;
+    }
+
+    if (!lcb_getenv_nonempty("LCB_LOGLEVEL", vbuf, sizeof(vbuf))) {
+        return NULL;
+    }
+
+    if (sscanf(vbuf, "%d", &lvl) != 1) {
+        return NULL;
+    }
+
+    if (!lvl) {
+        /** "0" */
+        return NULL;
+    }
+
+    /** The "lowest" level we can expose is WARN, e.g. ERROR-1 */
+    lvl = LCB_LOG_ERROR - lvl;
+    console_logprocs.minlevel = lvl;
+    return lcb_console_logprocs;
+}

data/ext/libcouchbase/src/logging.h

@@ -0,0 +1,88 @@
+/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *     Copyright 2014 Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#ifndef LCB_LOGGING_H
+#define LCB_LOGGING_H
+#include <stdarg.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif /* __cplusplus */
+
+
+struct lcb_settings_st;
+struct lcb_st;
+struct lcbvb_CONFIG_st;
+
+/**
+ * Default printf logger which is enabled via LCB_LOGLEVEL in the
+ * environment
+ */
+extern struct lcb_logprocs_st *lcb_console_logprocs;
+
+struct lcb_CONSOLELOGGER {
+    struct lcb_logprocs_st base;
+    FILE *fp;
+    int minlevel;
+};
+
+/**
+ * Log a message via the installed logger. The parameters correlate to the
+ * arguments passed to the lcb_logging_callback function.
+ *
+ * Typically a subsystem may wish to define macros in order to reduce the
+ * number of arguments manually passed for each message.
+ */
+LCB_INTERNAL_API
+void lcb_log(const struct lcb_settings_st *settings,
+             const char *subsys,
+             int severity,
+             const char *srcfile,
+             int srcline,
+             const char *fmt, ...)
+
+#ifdef __GNUC__
+             __attribute__ ((format(printf, 6, 7)))
+#endif
+             ;
+
+LCB_INTERNAL_API
+void lcb_log_badconfig(const struct lcb_settings_st *settings,
+    const char *subsys, int severity, const char *srcfile, int srcline,
+    const struct lcbvb_CONFIG_st *vbc, const char *origin_txt);
+
+lcb_logprocs * lcb_init_console_logger(void);
+
+#define LCB_LOGS(settings, subsys, severity, msg) \
+    lcb_log(settings, subsys, severity, __FILE__, __LINE__, msg)
+
+#define LCB_LOG_EX(settings, subsys, severity, msg) \
+    lcb_log(settings, subsys, severity, __FILE__, __LINE__, msg)
+
+#define LCB_LOG_BASIC(settings, msg) \
+    lcb_log(settings, "unknown", 0, __FILE__, __LINE__, msg)
+
+/** Macro for overcoming Win32 identifiers */
+#define LCB_LOG_ERR LCB_LOG_ERROR
+/** Undefine DEBUG macro to fix environments which are defining it */
+#undef DEBUG
+
+#ifdef __cplusplus
+}
+#endif /* __cplusplus */
+
+#endif /* LCB_LOGGING_H */

data/ext/libcouchbase/src/mc/compress.cc

@@ -0,0 +1,171 @@
+/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *     Copyright 2014-2018 Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#include "mcreq.h"
+#include "compress.h"
+
+#include <snappy.h>
+#include <snappy-sinksource.h>
+
+class FragBufSource : public snappy::Source
+{
+  public:
+    explicit FragBufSource(const lcb_FRAGBUF *buf_) : buf(buf_)
+    {
+        if (buf->total_length) {
+            left = buf->total_length;
+        } else {
+            left = 0;
+            for (unsigned int ii = 0; ii < buf->niov; ii++) {
+                left += buf->iov[ii].iov_len;
+            }
+        }
+        idx = 0;
+        ptr = static_cast< const char * >(buf->iov[idx].iov_base);
+    }
+
+    virtual ~FragBufSource() {}
+
+    virtual size_t Available() const
+    {
+        return left;
+    }
+
+    virtual const char *Peek(size_t *len)
+    {
+        *len =
+            buf->iov[idx].iov_len - static_cast< size_t >((ptr - static_cast< const char * >(buf->iov[idx].iov_base)));
+        return ptr;
+    }
+
+    virtual void Skip(size_t n)
+    {
+        do {
+            size_t spanleft = buf->iov[idx].iov_len - (ptr - static_cast< const char * >(buf->iov[idx].iov_base));
+            if (n < spanleft) {
+                ptr += n;
+                left -= n;
+                break;
+            }
+            if (idx + 1 >= buf->niov) {
+                left = 0;
+                ptr = NULL;
+                break;
+            }
+            left -= spanleft;
+            n -= spanleft;
+            ptr = static_cast< const char * >(buf->iov[++idx].iov_base);
+        } while (n > 0);
+        if (left == 0 || idx >= buf->niov) {
+            ptr = NULL;
+            left = 0;
+        }
+    }
+
+  private:
+    const lcb_FRAGBUF *buf;
+    const char *ptr;
+    size_t left;
+    unsigned int idx;
+};
+
+int mcreq_compress_value(mc_PIPELINE *pl, mc_PACKET *pkt, const lcb_VALBUF *vbuf, lcb_settings *settings,
+                         int *should_compress)
+{
+    size_t maxsize, compsize = 0, origsize = 0;
+
+    snappy::Source *source = NULL;
+    switch (vbuf->vtype) {
+        case LCB_KV_COPY:
+        case LCB_KV_CONTIG:
+            origsize = vbuf->u_buf.contig.nbytes;
+            if (origsize < settings->compress_min_size) {
+                *should_compress = 0;
+                mcreq_reserve_value(pl, pkt, vbuf);
+                return 0;
+            }
+            source = new snappy::ByteArraySource(static_cast< const char * >(vbuf->u_buf.contig.bytes),
+                                                 vbuf->u_buf.contig.nbytes);
+            break;
+
+        case LCB_KV_IOV:
+        case LCB_KV_IOVCOPY:
+            if (vbuf->u_buf.multi.total_length == 0) {
+                for (unsigned int ii = 0; ii < vbuf->u_buf.multi.niov; ii++) {
+                    origsize += vbuf->u_buf.multi.iov[ii].iov_len;
+                }
+            }
+            if (origsize < settings->compress_min_size) {
+                *should_compress = 0;
+                mcreq_reserve_value(pl, pkt, vbuf);
+                return 0;
+            }
+            source = new FragBufSource(&vbuf->u_buf.multi);
+            break;
+
+        default:
+            return -1;
+    }
+
+    maxsize = snappy::MaxCompressedLength(source->Available());
+    if (mcreq_reserve_value2(pl, pkt, maxsize) != LCB_SUCCESS) {
+        return -1;
+    }
+    nb_SPAN *outspan = &pkt->u_value.single;
+    snappy::UncheckedByteArraySink sink(SPAN_BUFFER(outspan));
+
+    Compress(source, &sink);
+    compsize = sink.CurrentDestination() - SPAN_BUFFER(outspan);
+    delete source;
+
+    if (compsize == 0 || (((float)compsize / origsize) > settings->compress_min_ratio)) {
+        netbuf_mblock_release(&pl->nbmgr, outspan);
+        *should_compress = 0;
+        mcreq_reserve_value(pl, pkt, vbuf);
+        return 0;
+    }
+
+    if (compsize < maxsize) {
+        /* chop off some bytes? */
+        nb_SPAN trailspan = *outspan;
+        trailspan.offset += compsize;
+        trailspan.size = maxsize - compsize;
+        netbuf_mblock_release(&pl->nbmgr, &trailspan);
+        outspan->size = compsize;
+    }
+    return 0;
+}
+
+int mcreq_inflate_value(const void *compressed, lcb_SIZE ncompressed, const void **bytes, lcb_SIZE *nbytes,
+                        void **freeptr)
+{
+    size_t compsize = 0;
+
+    if (!snappy::GetUncompressedLength(static_cast< const char * >(compressed), (size_t)ncompressed, &compsize)) {
+        return -1;
+    }
+    *freeptr = malloc(compsize);
+    if (!snappy::RawUncompress(static_cast< const char * >(compressed), ncompressed, static_cast< char * >(*freeptr))) {
+        free(*freeptr);
+        *freeptr = NULL;
+        return -1;
+    }
+
+    *bytes = *freeptr;
+    *nbytes = compsize;
+    return 0;
+}

data/ext/libcouchbase/src/mc/compress.h

@@ -0,0 +1,56 @@
+/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *     Copyright 2014 Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#include "mcreq.h"
+#include "settings.h"
+#ifndef LCB_MCCOMPRESS_H
+#define LCB_MCCOMPRESS_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Stores a compressed payload into a packet
+ * @param pl The pipeline which hosts the packet
+ * @param pkt The packet which hosts the value
+ * @param vbuf The user input to be compressed
+ * @param settings The instance settings
+ * @param should_compress The pointer, which stores zero if the value is not compressed
+ * @return 0 if successful, nonzero on error.
+ */
+int mcreq_compress_value(mc_PIPELINE *pl, mc_PACKET *pkt, const lcb_VALBUF *vbuf, lcb_settings *settings,
+                         int *should_compress);
+
+/**
+ * Inflate a compressed value
+ * @param compressed The value to inflate
+ * @param ncompressed Size of value to inflate
+ * @param[out] bytes The inflated value
+ * @param[out] nbytes The size of the inflated value
+ * @param[in/out] freeptr Pointer initialized to NULL (or an malloc'd buffer)
+ * which on output should point to a malloc'd buffer to be freed() when no
+ * longer required.
+ * @return 0 if successful, nonzero on error.
+ */
+int mcreq_inflate_value(const void *compressed, lcb_SIZE ncompressed, const void **bytes, lcb_SIZE *nbytes,
+                        void **freeptr);
+
+#ifdef __cplusplus
+}
+#endif
+#endif