libcouchbase 1.2.8 → 1.3.0

data/ext/libcouchbase/src/http/http_io.cc

@@ -23,10 +23,10 @@
 #include "sllist.h"
 #include <lcbio/ssl.h>
 
+#define LOGFMT CTX_LOGFMT
+#define LOGID(req) CTX_LOGID(req->ioctx)
 using namespace lcb::http;
 
-#define LOGFMT "<%s:%s>"
-#define LOGID(req) get_ctx_host((req)->ioctx), get_ctx_port((req)->ioctx)
 #define LOGARGS(req, lvl) req->instance->settings, "http-io", LCB_LOG_##lvl, __FILE__, __LINE__
 
 void
@@ -233,6 +233,23 @@ on_connected(lcbio_SOCKET *sock, void *arg, lcb_error_t err, lcbio_OSERR syserr)
     procs.cb_err = io_error;
     procs.cb_read = io_read;
     req->ioctx = lcbio_ctx_new(sock, arg, &procs);
+    switch (req->reqtype) {
+    case LCB_HTTP_TYPE_N1QL:
+        sock->service = LCBIO_SERVICE_N1QL;
+        break;
+    case LCB_HTTP_TYPE_VIEW:
+        sock->service = LCBIO_SERVICE_VIEW;
+        break;
+    case LCB_HTTP_TYPE_FTS:
+        sock->service = LCBIO_SERVICE_FTS;
+        break;
+    case LCB_HTTP_TYPE_CBAS:
+        sock->service = LCBIO_SERVICE_CBAS;
+        break;
+    default:
+        sock->service = LCBIO_SERVICE_MGMT;
+        break;
+    }
     req->ioctx->subsys = "mgmt/capi";
     lcbio_ctx_put(req->ioctx, &req->preamble[0], req->preamble.size());
     if (!req->body.empty()) {

data/ext/libcouchbase/src/instance.cc

@@ -19,13 +19,13 @@
 #include "connspec.h"
 #include "logging.h"
 #include "hostlist.h"
+#include "rnd.h"
 #include "http/http.h"
 #include "bucketconfig/clconfig.h"
 #include <lcbio/iotable.h>
 #include <lcbio/ssl.h>
 #define LOGARGS(obj,lvl) (obj)->settings, "instance", LCB_LOG_##lvl, __FILE__, __LINE__
 
-static volatile unsigned int lcb_instance_index = 0;
 using namespace lcb;
 
 LIBCOUCHBASE_API
@@ -56,6 +56,11 @@ LIBCOUCHBASE_API
 void
 lcb_set_auth(lcb_t instance, lcb_AUTHENTICATOR *auth)
 {
+    if (LCBT_SETTING(instance, keypath)) {
+        lcb_log(LOGARGS(instance, WARN),
+                "Custom authenticator ignored when SSL client certificate authentication in use");
+        return;
+    }
     /* First increase refcount in case they are the same object(!) */
     lcbauth_ref(auth);
     lcbauth_unref(instance->settings->auth);
@@ -74,7 +79,11 @@ lcb_st::add_bs_host(const char *host, int port, unsigned bstype)
         tname = "HTTP";
         target = ht_nodes;
     }
-    lcb_log(LOGARGS(this, DEBUG), "Adding host %s:%d to initial %s bootstrap list", host, port, tname);
+    bool ipv6 = strchr(host, ':') != NULL;
+    lcb_log(LOGARGS(this, DEBUG), "Adding host " LCB_LOG_SPEC("%s%s%s:%d") " to initial %s bootstrap list",
+            this->settings->log_redaction ? LCB_LOG_SD_OTAG : "",
+            ipv6 ? "[" : "", host, ipv6 ? "]" : "", port,
+            this->settings->log_redaction ? LCB_LOG_SD_CTAG : "", tname);
     target->add(host, port);
 }
 
@@ -114,6 +123,8 @@ lcb_st::populate_nodes(const Connspec& spec)
         const Spechost &dh = spec.hosts()[ii];
         add_bs_host(dh, defl_http, defl_cccp);
     }
+    lcb_log(LOGARGS(this, TRACE), "Bootstrap hosts loaded (cccp:%d, http:%d)", (int)mc_nodes->size(),
+            (int)ht_nodes->size());
 }
 
 lcb_error_t
@@ -148,7 +159,9 @@ lcb_st::process_dns_srv(Connspec& spec)
         sh.port = std::atoi(src.port);
         sh.type = spec.default_port();
         spec.add_host(sh);
-        lcb_log(LOGARGS(this, INFO), "Found host %s:%d via DNS SRV", sh.hostname.c_str(), sh.port);
+        bool ipv6 = sh.hostname.find(':') != std::string::npos;
+        lcb_log(LOGARGS(this, INFO), "Found host %s%s%s:%d via DNS SRV", ipv6 ? "[" : "", sh.hostname.c_str(),
+                ipv6 ? "]" : "", (int)sh.port);
     }
     delete hl;
 
@@ -240,6 +253,11 @@ setup_ssl(lcb_t obj, const Connspec& params)
         settings->certpath = strdup(optbuf);
     }
 
+    if (lcb_getenv_nonempty("LCB_SSL_KEY", optbuf, sizeof optbuf)) {
+        lcb_log(LOGARGS(obj, INFO), "SSL key %s specified on environment", optbuf);
+        settings->keypath = strdup(optbuf);
+    }
+
     if (lcb_getenv_nonempty("LCB_SSL_MODE", optbuf, sizeof optbuf)) {
         if (sscanf(optbuf, "%d", &env_policy) != 1) {
             lcb_log(LOGARGS(obj, ERR), "Invalid value for environment LCB_SSL. (%s)", optbuf);
@@ -250,10 +268,18 @@ setup_ssl(lcb_t obj, const Connspec& params)
         }
     }
 
+    if (settings->truststorepath == NULL && !params.truststorepath().empty()) {
+        settings->truststorepath = strdup(params.truststorepath().c_str());
+    }
+
     if (settings->certpath == NULL && !params.certpath().empty()) {
         settings->certpath = strdup(params.certpath().c_str());
     }
 
+    if (settings->keypath == NULL && !params.keypath().empty()) {
+        settings->keypath = strdup(params.keypath().c_str());
+    }
+
     if (env_policy == -1) {
         settings->sslopts = params.sslopts();
     }
@@ -264,11 +290,21 @@ setup_ssl(lcb_t obj, const Connspec& params)
         } else {
             lcb_log(LOGARGS(obj, INFO), "ssl=no_global_init. Not initializing openssl globals");
         }
-        settings->ssl_ctx = lcbio_ssl_new(settings->certpath,
-            settings->sslopts & LCB_SSL_NOVERIFY, &err, settings);
+        if (settings->keypath && !settings->certpath) {
+            lcb_log(LOGARGS(obj, ERR), "SSL key have to be specified with certificate");
+            return LCB_EINVAL;
+        }
+        settings->ssl_ctx =
+            lcbio_ssl_new(settings->truststorepath, settings->certpath, settings->keypath,
+                          settings->sslopts & LCB_SSL_NOVERIFY, &err, settings);
         if (!settings->ssl_ctx) {
             return err;
         }
+    } else {
+        // keypath might be used to flag, that library is using SSL authentication
+        // To avoid skipping other authentication mechanisms, cleanup the keypath.
+        free(settings->keypath);
+        settings->keypath = NULL;
     }
     return LCB_SUCCESS;
 }
@@ -329,7 +365,6 @@ lcb_reinit3(lcb_t obj, const char *connstr)
     Connspec params;
     lcb_error_t err;
     const char *errmsg = NULL;
-    memset(&params, 0, sizeof params);
     err = params.parse(connstr, &errmsg);
 
     if (err != LCB_SUCCESS) {
@@ -367,6 +402,10 @@ lcb_error_t lcb_create(lcb_t *instance,
     lcb_error_t err;
     lcb_settings *settings;
 
+#if !defined(COMPILER_SUPPORTS_CXX11) || (defined(_MSC_VER) && _MSC_VER < 1600)
+    lcb_rnd_global_init();
+#endif
+
     if (options) {
         io_priv = options->v.v0.io;
         if (options->version > 0) {
@@ -385,6 +424,7 @@ lcb_error_t lcb_create(lcb_t *instance,
         err = LCB_CLIENT_ENOMEM;
         goto GT_DONE;
     }
+    obj->crypto = new std::map<std::string, lcbcrypto_PROVIDER*>();
     if (!(settings = lcb_settings_new())) {
         err = LCB_CLIENT_ENOMEM;
         goto GT_DONE;
@@ -394,6 +434,7 @@ lcb_error_t lcb_create(lcb_t *instance,
     obj->type = type;
     obj->settings = settings;
     obj->settings->conntype = type;
+    obj->settings->ipv6 = spec.ipv6_policy();
 
     settings->bucket = strdup(spec.bucket().c_str());
 
@@ -411,14 +452,21 @@ lcb_error_t lcb_create(lcb_t *instance,
     }
 
     settings->logger = lcb_init_console_logger();
-    settings->iid = lcb_instance_index++;
+    settings->iid = lcb_next_rand32();
     if (spec.loglevel()) {
         lcb_U32 val = spec.loglevel();
         lcb_cntl(obj, LCB_CNTL_SET, LCB_CNTL_CONLOGGER_LEVEL, &val);
     }
+    settings->log_redaction = spec.logredact();
+    if (settings->log_redaction) {
+        lcb_log(LOGARGS(obj, INFO), "Logging redaction enabled. Logs have reduced identifying information. Diagnosis "
+                                    "and support of issues may be challenging or not possible in this configuration");
+    }
 
     lcb_log(LOGARGS(obj, INFO), "Version=%s, Changeset=%s", lcb_get_version(NULL), LCB_VERSION_CHANGESET);
-    lcb_log(LOGARGS(obj, INFO), "Effective connection string: %s. Bucket=%s", spec.connstr().c_str(), settings->bucket);
+    lcb_log(LOGARGS(obj, INFO), "Effective connection string: " LCB_LOG_SPEC("%s") ". Bucket=" LCB_LOG_SPEC("%s"),
+            settings->log_redaction ? LCB_LOG_SD_OTAG : "", spec.connstr().c_str(), settings->log_redaction ? LCB_LOG_SD_CTAG : "",
+            settings->log_redaction ? LCB_LOG_MD_OTAG : "", settings->bucket, settings->log_redaction ? LCB_LOG_MD_CTAG : "");
 
     if (io_priv == NULL) {
         lcb_io_opt_t ops;
@@ -443,7 +491,7 @@ lcb_error_t lcb_create(lcb_t *instance,
         obj->http_sockpool->set_options(pool_opts);
     }
 
-    obj->confmon = new clconfig::Confmon(settings, obj->iotable);
+    obj->confmon = new clconfig::Confmon(settings, obj->iotable, obj);
     obj->ht_nodes = new Hostlist();
     obj->mc_nodes = new Hostlist();
     obj->retryq = new RetryQueue(&obj->cmdq, obj->iotable, obj->settings);
@@ -470,11 +518,11 @@ lcb_error_t lcb_create(lcb_t *instance,
     if ((err = init_providers(obj, spec)) != LCB_SUCCESS) {
         goto GT_DONE;
     }
-
-    if (err != LCB_SUCCESS) {
-        lcb_destroy(obj);
-        return err;
+#ifdef LCB_TRACING
+    if (settings->use_tracing) {
+        settings->tracer = lcbtrace_new(obj, LCBTRACE_F_THRESHOLD);
     }
+#endif
 
     obj->last_error = err;
     GT_DONE:
@@ -487,6 +535,12 @@ lcb_error_t lcb_create(lcb_t *instance,
     return err;
 }
 
+LIBCOUCHBASE_API
+int lcb_is_redacting_logs(lcb_t instance)
+{
+    return instance && instance->settings && instance->settings->log_redaction;
+}
+
 typedef struct {
     lcbio_pTABLE table;
     lcbio_pTIMER timer;
@@ -586,6 +640,13 @@ void lcb_destroy(lcb_t instance)
         instance->scratch = NULL;
     }
 
+    for (std::map< std::string, lcbcrypto_PROVIDER * >::iterator ii = instance->crypto->begin();
+         ii != instance->crypto->end(); ++ii) {
+        lcbcrypto_unref(ii->second);
+    }
+    delete instance->crypto;
+    instance->crypto = NULL;
+
     delete[] instance->dcpinfo;
     memset(instance, 0xff, sizeof(*instance));
     free(instance);
@@ -721,13 +782,16 @@ LIBCOUCHBASE_API
 int
 lcb_supports_feature(int n)
 {
-    if (n == LCB_SUPPORTS_SNAPPY) {
-#ifdef LCB_NO_SNAPPY
-        return 0;
-#else
+    if (n == LCB_SUPPORTS_TRACING) {
+#ifdef LCB_TRACING
         return 1;
+#else
+        return 0;
 #endif
     }
+    if (n == LCB_SUPPORTS_SNAPPY) {
+        return 1;
+    }
     if (n == LCB_SUPPORTS_SSL) {
         return lcbio_ssl_supported();
     } else {
@@ -815,6 +879,15 @@ const char *lcb_strerror_short(lcb_error_t error)
     return "<FIXME: Not an LCB error>";
 }
 
+LCB_INTERNAL_API
+const char *lcb_strerror_long(lcb_error_t error)
+{
+#define X(c, v, t, s) if (error == c) { return #c " (" #v "): " s; }
+    LCB_XERR(X)
+#undef X
+    return "<FIXME: Not an LCB error>";
+}
+
 LIBCOUCHBASE_API
 int lcb_get_errtype(lcb_error_t err)
 {

data/ext/libcouchbase/src/internal.h

@@ -34,6 +34,7 @@
 #include <libcouchbase/vbucket.h>
 #include <libcouchbase/api3.h>
 #include <libcouchbase/pktfwd.h>
+#include <libcouchbase/crypto.h>
 
 /* Internal dependencies */
 #include <lcbio/lcbio.h>
@@ -51,6 +52,8 @@
 /* n1ql cache */
 #include "n1ql/n1ql-internal.h"
 
+#include "tracing/tracing-internal.h"
+
 #include "hostlist.h"
 
 #ifdef __cplusplus
@@ -135,6 +138,8 @@ struct lcb_st {
     lcb_BTYPE btype; /**< Type of the bucket */
 
     #ifdef __cplusplus
+    typedef std::map<std::string, lcbcrypto_PROVIDER *> lcb_ProviderMap;
+    lcb_ProviderMap *crypto;
     lcb_settings* getSettings() { return settings; }
     lcbio_pTABLE getIOT() { return iotable; }
     inline void add_bs_host(const char *host, int port, unsigned bstype);

data/ext/libcouchbase/src/lcbio/connect.cc

@@ -22,6 +22,7 @@
 #include "settings.h"
 #include "timer-ng.h"
 #include "timer-cxx.h"
+#include "rnd.h"
 #include <errno.h>
 
 using namespace lcb::io;
@@ -32,15 +33,22 @@ using namespace lcb::io;
 #endif
 #define LOGARGS(conn, lvl) conn->settings, "connection", LCB_LOG_##lvl, __FILE__, __LINE__
 static const lcb_host_t *get_loghost(lcbio_SOCKET *s) {
-    static lcb_host_t host = { "NOHOST", "NOPORT" };
+    static lcb_host_t host = {"NOHOST", "NOPORT", 0};
     if (!s) { return &host; }
     if (!s->info) { return &host; }
     return &s->info->ep;
 }
 
 /** Format string arguments for %p%s:%s */
-#define CSLOGID(sock) get_loghost(sock)->host, get_loghost(sock)->port, (void*)sock
-#define CSLOGFMT "<%s:%s> (SOCK=%p) "
+#define CSLOGID(sock)                                           \
+    sock->settings->log_redaction ? LCB_LOG_SD_OTAG : "",       \
+        get_loghost(sock)->ipv6 ? "[" : "",                     \
+        get_loghost(sock)->host,                                \
+        get_loghost(sock)->ipv6 ? "]" : "",                     \
+        get_loghost(sock)->port,                                \
+        sock->settings->log_redaction ? LCB_LOG_SD_CTAG : "",   \
+        sock->id
+#define CSLOGFMT "<" LCB_LOG_SPEC("%s%s%s:%s") "> (SOCK=%016" PRIx64 ") "
 
 #define LOGARGS_T(lvl) LOGARGS(this->sock, lvl)
 #define CSLOGID_T() CSLOGID(this->sock)
@@ -356,7 +364,8 @@ C_conncb(lcb_sockdata_t *sock, int status)
     lcbio_SOCKET *s = reinterpret_cast<lcbio_SOCKET*>(sock->lcbconn);
     Connstart *cs = reinterpret_cast<Connstart*>(s->ctx);
 
-    lcb_log(LOGARGS(s, TRACE), CSLOGFMT "Received completion handler. Status=%d. errno=%d", CSLOGID(s), status, IOT_ERRNO(s->io));
+    lcb_log(LOGARGS(s, TRACE), CSLOGFMT "Received completion handler. Status=%d. errno=%d [%s]", CSLOGID(s), status,
+            IOT_ERRNO(s->io), strerror(IOT_ERRNO(s->io)));
 
     if (!--s->refcount) {
         lcbio__destroy(s);
@@ -449,6 +458,7 @@ Connstart::Connstart(lcbio_TABLE* iot_, lcb_settings* settings_,
     sock->settings = settings_;
     sock->ctx = this;
     sock->refcount = 1;
+    sock->id = lcb_next_rand64();
     sock->info = reinterpret_cast<lcbio_CONNINFO*>(calloc(1, sizeof(*sock->info)));
     sock->info->ep = *dest;
     lcbio_table_ref(sock->io);
@@ -529,6 +539,7 @@ lcbio_wrap_fd(lcbio_pTABLE iot, lcb_settings *settings, lcb_socket_t fd)
     ret->io = iot;
     ret->refcount = 1;
     ret->u.fd = fd;
+    ret->id = lcb_next_rand64();
 
     lcbio_table_ref(ret->io);
     lcb_settings_ref(ret->settings);
@@ -566,3 +577,27 @@ lcbio__destroy(lcbio_SOCKET *s)
     lcb_settings_unref(s->settings);
     free(s);
 }
+
+const char * lcbio_svcstr(lcbio_SERVICE service)
+{
+    switch (service) {
+    case LCBIO_SERVICE_CFG:
+        return "config";
+    case LCBIO_SERVICE_KV:
+        return "kv";
+    case LCBIO_SERVICE_MGMT:
+        return "mgmt";
+    case LCBIO_SERVICE_VIEW:
+        return "view";
+    case LCBIO_SERVICE_N1QL:
+        return "n1ql";
+    case LCBIO_SERVICE_FTS:
+        return "fts";
+    case LCBIO_SERVICE_CBAS:
+        return "cbas";
+    case LCBIO_SERVICE_UNSPEC:
+        /* fallthrough */
+    default:
+        return "unspec";
+    }
+}

data/ext/libcouchbase/src/lcbio/connect.h

@@ -75,12 +75,29 @@ typedef struct {
     lcb_host_t ep;
 } lcbio_CONNINFO;
 
+struct lcb_IOMETRICS_st;
+
+/** @brief Subsystem, which utilizes the socket */
+typedef enum {
+    LCBIO_SERVICE_UNSPEC = 0,
+    LCBIO_SERVICE_CFG,
+    LCBIO_SERVICE_KV,
+    LCBIO_SERVICE_MGMT,
+    LCBIO_SERVICE_VIEW,
+    LCBIO_SERVICE_N1QL,
+    LCBIO_SERVICE_FTS,
+    LCBIO_SERVICE_CBAS,
+    LCBIO_SERVICE_MAX
+} lcbio_SERVICE;
+
+const char * lcbio_svcstr(lcbio_SERVICE service);
 
 /** @brief Core socket structure */
 typedef struct lcbio_SOCKET {
     lcbio_pTABLE io;
     lcb_settings *settings;
     void *ctx;
+    struct lcb_IOMETRICS_st *metrics;
     lcbio_CONNINFO *info;
     lcbio_OSERR last_error; /**< last OS error */
     unsigned refcount; /**< refcount on socket */
@@ -89,6 +106,9 @@ typedef struct lcbio_SOCKET {
         lcb_socket_t fd;
     } u;
     lcb_list_t protos;
+    hrtime_t atime;
+    lcbio_SERVICE service;
+    lcb_U64 id;
 } lcbio_SOCKET;
 
 
@@ -345,6 +365,13 @@ lcbio_table_unref(lcbio_pTABLE iot);
 void
 lcbio_table_ref(lcbio_pTABLE iot);
 
+/**
+ * Set the metrics object for the socket. Various operations will then log
+ * the number of bytes written/received on the socket.
+ */
+#define \
+    lcbio_set_metrics(sock, m) (sock)->metrics = m
+
 /** @}*/
 
 /** @name IO Status Codes