libcouchbase 1.2.8 → 1.3.0

data/ext/libcouchbase/tools/cbc-proxy.cc

@@ -16,13 +16,14 @@
 
 #define LCB_NO_DEPR_CXX_CTORS
 
-#include "common/my_inttypes.h"
 #include "config.h"
 #include <sys/types.h>
 #include <libcouchbase/couchbase.h>
 #include <libcouchbase/vbucket.h>
 #include <libcouchbase/api3.h>
 #include <libcouchbase/pktfwd.h>
+#include <libcouchbase/n1ql.h>
+#include <libcouchbase/cbft.h>
 #include <memcached/protocol_binary.h>
 #include <iostream>
 #include <iomanip>
@@ -61,6 +62,8 @@ static lcb_t instance = NULL;
 static struct event_base *evbase = NULL;
 static Histogram hg;
 
+static char app_client_string[] = "cbc-proxy";
+
 #define LOGARGS(lvl) (instance)->settings, "proxy", LCB_LOG_##lvl, __FILE__, __LINE__
 #define CL_LOGFMT "<%s:%s> (cl=%p,fd=%d) "
 #define CL_LOGID(cl) cl->host, cl->port, (void *)cl, cl->fd
@@ -152,9 +155,10 @@ struct client {
     struct bufferevent *bev;
     char host[NI_MAXHOST + 1];
     char port[NI_MAXSERV + 1];
+    long cnt;
 };
 
-static void dump_bytes(const struct client *cl, const char *msg, void *ptr, size_t len)
+static void dump_bytes(const struct client *cl, const char *msg, const void *ptr, size_t len)
 {
     if (!config.isTrace()) {
         return;
@@ -246,6 +250,50 @@ static void pktfwd_callback(lcb_t, const void *cookie, lcb_error_t err, lcb_PKTF
     }
 }
 
+extern "C" {
+#define DEFINE_ROW_CALLBACK(cbname, resptype)                                                                          \
+    static void cbname(lcb_t, int, const resptype *resp)                                                               \
+    {                                                                                                                  \
+        char key[100] = {0};                                                                                           \
+        size_t nkey;                                                                                                   \
+        struct client *cl = (struct client *)resp->cookie;                                                             \
+                                                                                                                       \
+        protocol_binary_response_header header = {};                                                                   \
+        header.response.magic = PROTOCOL_BINARY_RES;                                                                   \
+        header.response.opcode = PROTOCOL_BINARY_CMD_STAT;                                                             \
+                                                                                                                       \
+        struct evbuffer *output = bufferevent_get_output(cl->bev);                                                     \
+                                                                                                                       \
+        if (resp->rflags & LCB_RESP_F_FINAL) {                                                                         \
+            memcpy(key, "meta", 4);                                                                                    \
+        } else {                                                                                                       \
+            snprintf(key, sizeof(key), "row-%ld", cl->cnt++);                                                          \
+        }                                                                                                              \
+        nkey = strlen(key);                                                                                            \
+        header.response.keylen = htons(nkey);                                                                          \
+        header.response.bodylen = htonl(resp->nrow + nkey);                                                            \
+                                                                                                                       \
+        evbuffer_expand(output, resp->nrow + sizeof(header.bytes));                                                    \
+        dump_bytes(cl, "response", header.bytes, sizeof(header.bytes));                                                \
+        evbuffer_add(output, header.bytes, sizeof(header.bytes));                                                      \
+        dump_bytes(cl, "response", key, nkey);                                                                         \
+        evbuffer_add(output, key, nkey);                                                                               \
+        dump_bytes(cl, "response", resp->row, resp->nrow);                                                             \
+        evbuffer_add(output, resp->row, resp->nrow);                                                                   \
+                                                                                                                       \
+        if (resp->rflags & LCB_RESP_F_FINAL) {                                                                         \
+            header.response.keylen = 0;                                                                                \
+            header.response.bodylen = 0;                                                                               \
+            evbuffer_expand(output, sizeof(header.bytes));                                                             \
+            dump_bytes(cl, "response", header.bytes, sizeof(header.bytes));                                            \
+            evbuffer_add(output, header.bytes, sizeof(header.bytes));                                                  \
+        }                                                                                                              \
+    }
+
+DEFINE_ROW_CALLBACK(n1ql_callback, lcb_RESPN1QL)
+DEFINE_ROW_CALLBACK(fts_callback, lcb_RESPFTS)
+}
+
 static void conn_readcb(struct bufferevent *bev, void *cookie)
 {
     struct client *cl = (struct client *)cookie;
@@ -273,12 +321,67 @@ static void conn_readcb(struct bufferevent *bev, void *cookie)
     evbuffer_remove(input, pkt, pktlen);
 
     lcb_sched_enter(instance);
+    dump_bytes(cl, "request", pkt, pktlen);
+    if (header.request.opcode == PROTOCOL_BINARY_CMD_STAT) {
+        lcb_U8 extlen = ntohs(header.request.extlen);
+        lcb_U16 keylen = ntohs(header.request.keylen);
+        if (keylen < 5) {
+            goto FWD;
+        }
+        char *key = (char *)pkt + sizeof(header) + extlen;
+        lcb_error_t rc;
+        int cbas = memcmp(key, "cbas ", 5) == 0;
+        if (cbas || memcmp(key, "n1ql ", 5) == 0) {
+            lcb_N1QLPARAMS *nparams = lcb_n1p_new();
+
+            rc = lcb_n1p_setquery(nparams, key + 5, keylen - 5, LCB_N1P_QUERY_STATEMENT);
+            if (rc != LCB_SUCCESS) {
+                lcb_log(LOGARGS(INFO), CL_LOGFMT "failed to set query for %s", CL_LOGID(cl),
+                        cbas ? "analytics" : "N1QL");
+                goto FWD;
+            }
+            lcb_CMDN1QL cmd = {0};
+            rc = lcb_n1p_mkcmd(nparams, &cmd);
+            if (rc != LCB_SUCCESS) {
+                lcb_log(LOGARGS(INFO), CL_LOGFMT "failed to generate %s command", CL_LOGID(cl),
+                        cbas ? "analytics" : "N1QL");
+                goto FWD;
+            }
+            cmd.callback = n1ql_callback;
+            cl->cnt = 0;
+            if (cbas) {
+                cmd.cmdflags |= LCB_CMDN1QL_F_CBASQUERY;
+            }
+            rc = lcb_n1ql_query(instance, cl, &cmd);
+            if (rc != LCB_SUCCESS) {
+                lcb_log(LOGARGS(INFO), CL_LOGFMT "failed to schedule %s command", CL_LOGID(cl),
+                        cbas ? "analytics" : "N1QL");
+                goto FWD;
+            }
+            lcb_n1p_free(nparams);
+            goto DONE;
+        } else if (memcmp(key, "fts ", 4) == 0) {
+            lcb_CMDFTS cmd = {0};
+            cmd.query = key + 4;
+            cmd.nquery = keylen - 4;
+            rc = lcb_fts_query(instance, cl, &cmd);
+            cmd.callback = fts_callback;
+            cl->cnt = 0;
+            if (rc != LCB_SUCCESS) {
+                lcb_log(LOGARGS(INFO), CL_LOGFMT "failed to schedule FTS command", CL_LOGID(cl));
+                goto FWD;
+            }
+            goto DONE;
+        }
+    }
+FWD : {
     lcb_CMDPKTFWD cmd = {0};
     cmd.vb.vtype = LCB_KV_COPY;
     cmd.vb.u_buf.contig.bytes = pkt;
     cmd.vb.u_buf.contig.nbytes = pktlen;
-    dump_bytes(cl, "request", pkt, pktlen);
     good_or_die(lcb_pktfwd3(instance, cl, &cmd), "Failed to forward packet");
+}
+DONE:
     lcb_sched_leave(instance);
 }
 
@@ -350,6 +453,26 @@ static void sigint_handler(int)
     }
 }
 
+static void diag_callback(lcb_t, int, const lcb_RESPBASE *rb)
+{
+    const lcb_RESPDIAG *resp = (const lcb_RESPDIAG *)rb;
+    if (resp->rc != LCB_SUCCESS) {
+        fprintf(stderr, "failed: %s\n", lcb_strerror(NULL, resp->rc));
+    } else {
+        if (resp->njson) {
+            fprintf(stderr, "\n%.*s", (int)resp->njson, resp->json);
+        }
+    }
+}
+
+static void sigquit_handler(int)
+{
+    lcb_CMDDIAG req = {};
+    req.options = LCB_PINGOPT_F_JSONPRETTY;
+    req.id = app_client_string;
+    lcb_diag(instance, NULL, &req);
+}
+
 static void real_main(int argc, char **argv)
 {
     Parser parser;
@@ -372,8 +495,10 @@ static void real_main(int argc, char **argv)
 
     good_or_die(lcb_create(&instance, &cropts), "Failed to create connection");
     config.doCtls();
+    lcb_cntl(instance, LCB_CNTL_SET, LCB_CNTL_CLIENT_STRING, app_client_string);
     lcb_set_bootstrap_callback(instance, bootstrap_callback);
     lcb_set_pktfwd_callback(instance, pktfwd_callback);
+    lcb_install_callback3(instance, LCB_CALLBACK_DIAG, diag_callback);
 
     good_or_die(lcb_connect(instance), "Failed to connect to cluster");
     if (config.useTimings()) {
@@ -388,6 +513,12 @@ static void real_main(int argc, char **argv)
     action.sa_flags = 0;
     sigaction(SIGINT, &action, NULL);
 
+    /* setup CTRL-\ handler */
+    sigemptyset(&action.sa_mask);
+    action.sa_handler = sigquit_handler;
+    action.sa_flags = 0;
+    sigaction(SIGQUIT, &action, NULL);
+
     event_base_dispatch(evbase);
 }
 

data/ext/libcouchbase/tools/cbc-subdoc.cc

@@ -16,7 +16,6 @@
 
 #define LCB_NO_DEPR_CXX_CTORS
 
-#include "common/my_inttypes.h"
 #include "config.h"
 #include <sys/types.h>
 #include <libcouchbase/couchbase.h>
@@ -488,7 +487,7 @@ class UpsertHandler : public Handler
         }
         // currently it is not possible to upsert document without XATTRs
         // so lets allocate "_cbc" object with some useful stuff
-        std::string ver = "\"libcouchbase/" LCB_VERSION_STRING "\"";
+        std::string ver = "\"" LCB_CLIENT_ID "\"";
         std::string path = "_cbc.version";
 
         std::string key = args[0];

data/ext/libcouchbase/tools/cbc.cc

@@ -1,5 +1,4 @@
 #define NOMINMAX
-#include "common/my_inttypes.h"
 #include <map>
 #include <sstream>
 #include <iostream>
@@ -21,6 +20,7 @@
 #ifndef LCB_NO_SSL
 #include <openssl/crypto.h>
 #endif
+#include <snappy-stubs-public.h>
 
 using namespace cbc;
 
@@ -77,8 +77,17 @@ get_callback(lcb_t, lcb_CALLBACKTYPE cbtype, const lcb_RESPGET *resp)
 {
     string key = getRespKey((const lcb_RESPBASE *)resp);
     if (resp->rc == LCB_SUCCESS) {
-        fprintf(stderr, "%-20s CAS=0x%" PRIx64 ", Flags=0x%x. Size=%lu\n",
-            key.c_str(), resp->cas, resp->itmflags, (unsigned long)resp->nvalue);
+        fprintf(stderr, "%-20s CAS=0x%" PRIx64 ", Flags=0x%x, Size=%lu, Datatype=0x%02x",
+                key.c_str(), resp->cas, resp->itmflags, (unsigned long)resp->nvalue,
+                (int)resp->datatype);
+        if (resp->datatype) {
+            fprintf(stderr, "(");
+            if (resp->datatype & LCB_VALUE_F_JSON) {
+                fprintf(stderr, "JSON");
+            }
+            fprintf(stderr, ")");
+        }
+        fprintf(stderr, "\n");
         fflush(stderr);
         fwrite(resp->value, 1, resp->nvalue, stdout);
         fflush(stdout);
@@ -222,6 +231,34 @@ stats_callback(lcb_t, lcb_CALLBACKTYPE, const lcb_RESPSTATS *resp)
     fprintf(stdout, "\n");
 }
 
+static void
+watch_callback(lcb_t, lcb_CALLBACKTYPE, const lcb_RESPSTATS *resp)
+{
+    if (resp->rc != LCB_SUCCESS) {
+        fprintf(stderr, "ERROR 0x%02X (%s)\n", resp->rc, lcb_strerror(NULL, resp->rc));
+        return;
+    }
+    if (resp->server == NULL || resp->key == NULL) {
+        return;
+    }
+
+    string key = getRespKey((const lcb_RESPBASE *)resp);
+    if (resp->nvalue >  0) {
+        char *nptr = NULL;
+        uint64_t val =
+#ifdef _WIN32
+            _strtoi64
+#else
+            strtoll
+#endif
+            ((const char *)resp->value, &nptr, 10);
+        if (nptr != (const char *)resp->value) {
+            map<string, int64_t> *entry = reinterpret_cast< map<string, int64_t> *>(resp->cookie);
+            (*entry)[key] += val;
+        }
+    }
+}
+
 static void
 common_server_callback(lcb_t, int cbtype, const lcb_RESPSERVERBASE *sbase)
 {
@@ -372,17 +409,17 @@ Handler::run()
     lcb_error_t err;
     err = lcb_create(&instance, &cropts);
     if (err != LCB_SUCCESS) {
-        throw LcbError(err);
+        throw LcbError(err, "Failed to create instance");
     }
     params.doCtls(instance);
     err = lcb_connect(instance);
     if (err != LCB_SUCCESS) {
-        throw LcbError(err);
+        throw LcbError(err, "Failed to connect instance");
     }
     lcb_wait(instance);
     err = lcb_get_bootstrap_status(instance);
     if (err != LCB_SUCCESS) {
-        throw LcbError(err);
+        throw LcbError(err, "Failed to bootstrap instance");
     }
 
     if (params.useTimings()) {
@@ -508,8 +545,7 @@ SetHandler::addOptions()
     if (!hasFileList()) {
         parser.addOption(o_value);
     }
-    // This may be enabled again if datatype support is re-added
-    // parser.addOption(o_json);
+    parser.addOption(o_json);
 }
 
 lcb_storage_t
@@ -846,6 +882,22 @@ VersionHandler::run()
     } else {
         printf("  SSL: NOT SUPPORTED\n");
     }
+    if (lcb_supports_feature(LCB_SUPPORTS_SNAPPY)) {
+#define EXPAND(VAR)   VAR ## 1
+#define IS_EMPTY(VAR) EXPAND(VAR)
+
+#if defined(SNAPPY_MAJOR) && (IS_EMPTY(SNAPPY_MAJOR) != 1)
+        printf("  Snappy: %d.%d.%d\n", SNAPPY_MAJOR, SNAPPY_MINOR, SNAPPY_PATCHLEVEL);
+#else
+        printf("  Snappy: unknown\n");
+#endif
+    } else {
+        printf("  Snappy: NOT SUPPORTED\n");
+    }
+    printf("  Tracing: %sSUPPORTED\n", lcb_supports_feature(LCB_SUPPORTS_TRACING) ? "" : "NOT ");
+    printf("  System: %s; %s\n", LCB_SYSTEM, LCB_SYSTEM_PROCESSOR);
+    printf("  CC: %s; %s\n", LCB_C_COMPILER, LCB_C_FLAGS);
+    printf("  CXX: %s; %s\n", LCB_CXX_COMPILER, LCB_CXX_FLAGS);
 }
 
 void
@@ -898,6 +950,57 @@ StatsHandler::run()
     lcb_wait(instance);
 }
 
+void
+WatchHandler::run()
+{
+    Handler::run();
+    lcb_install_callback3(instance, LCB_CALLBACK_STATS, (lcb_RESPCALLBACK)watch_callback);
+    vector<string> keys = parser.getRestArgs();
+    if (keys.empty()) {
+        keys.push_back("cmd_total_ops");
+        keys.push_back("cmd_total_gets");
+        keys.push_back("cmd_total_sets");
+    }
+    int interval = o_interval.result();
+
+    map<string, int64_t> prev;
+
+    bool first = true;
+    while (true) {
+        map<string, int64_t> entry;
+        lcb_sched_enter(instance);
+        lcb_CMDSTATS cmd = { 0 };
+        lcb_error_t err = lcb_stats3(instance, (void *)&entry, &cmd);
+        if (err != LCB_SUCCESS) {
+            throw LcbError(err);
+        }
+        lcb_sched_leave(instance);
+        lcb_wait(instance);
+        if (first) {
+            for (vector<string>::iterator it = keys.begin(); it != keys.end(); ++it) {
+                fprintf(stderr, "%s: %" PRId64 "\n", it->c_str(), entry[*it]);
+            }
+            first = false;
+        } else {
+#ifndef _WIN32
+            if (isatty(STDERR_FILENO)) {
+                fprintf(stderr, "\033[%dA", (int)keys.size());
+            }
+#endif
+            for (vector<string>::iterator it = keys.begin(); it != keys.end(); ++it) {
+                fprintf(stderr, "%s: %" PRId64 "%20s\n", it->c_str(), (entry[*it] - prev[*it]) / interval, "");
+            }
+        }
+        prev = entry;
+#ifdef _WIN32
+        Sleep(interval * 1000);
+#else
+        sleep(interval);
+#endif
+    }
+}
+
+
 void
 VerbosityHandler::run()
 {
@@ -1572,6 +1675,7 @@ static const char* optionsOrder[] = {
         "write-config",
         "strerror",
         "ping",
+        "watch",
         NULL
 };
 
@@ -1640,6 +1744,7 @@ setupHandlers()
     handlers_s["rm"] = new RemoveHandler();
     handlers_s["cp"] = new SetHandler("cp");
     handlers_s["stats"] = new StatsHandler();
+    handlers_s["watch"] = new WatchHandler();
     handlers_s["verbosity"] = new VerbosityHandler();
     handlers_s["ping"] = new PingHandler();
     handlers_s["mcflush"] = new McFlushHandler();

data/ext/libcouchbase/tools/common/histogram.cc

@@ -12,6 +12,7 @@ Histogram::install(lcb_t inst, FILE *out)
     rc = lcb_cntl(inst, LCB_CNTL_GET, LCB_CNTL_KVTIMINGS, &hg);
     assert(rc == LCB_SUCCESS);
     assert(hg != NULL);
+    (void)rc;
 }
 
 void

data/ext/libcouchbase/tools/common/options.cc

@@ -77,9 +77,11 @@ ConnParams::ConnParams() :
     o_transport.description("Bootstrap protocol").argdesc("HTTP|CCCP|ALL").setDefault("ALL");
     o_configcache.description("Path to cached configuration");
     o_ssl.description("Enable SSL settings").argdesc("ON|OFF|NOVERIFY").setDefault("off");
-    o_certpath.description("Path to server certificate");
+    o_certpath.description("Path to server SSL certificate");
+    o_keypath.description("Path to client SSL private key");
     o_verbose.description("Set debugging output (specify multiple times for greater verbosity");
     o_dump.description("Dump verbose internal state after operations are done");
+    o_compress.description("Turn on compression of outgoing data (second time to force compression)").setDefault(false);
 
     o_cparams.description("Additional options for connection. "
         "Use -Dtimeout=SECONDS for KV operation timeout");
@@ -218,6 +220,8 @@ ConnParams::loadFileDefaults()
             o_connstr.setDefault(value).setPassed();
         } else if (key == "certpath") {
             o_certpath.setDefault(value).setPassed();
+        } else if (key == "keypath") {
+            o_keypath.setDefault(value).setPassed();
         } else if (key == "ssl") {
             o_ssl.setDefault(value).setPassed();
         } else {
@@ -259,7 +263,9 @@ ConnParams::writeConfig(const string& s)
     writeOption(f, o_user, "user");
     writeOption(f, o_passwd, "password");
     writeOption(f, o_ssl, "ssl");
+    writeOption(f, o_truststorepath, "truststorepath");
     writeOption(f, o_certpath, "certpath");
+    writeOption(f, o_keypath, "keypath");
 
     if (o_timeout.passed()) {
         f << "timeout=" << std::dec << o_timeout.result() << endl;
@@ -317,11 +323,21 @@ ConnParams::fillCropts(lcb_create_st& cropts)
         fprintf(stderr, "  Specifying the default port (8091) has no effect\n");
     }
 
+    if (o_truststorepath.passed()) {
+        connstr += "truststorepath=";
+        connstr += o_truststorepath.result();
+        connstr += '&';
+    }
     if (o_certpath.passed()) {
         connstr += "certpath=";
         connstr += o_certpath.result();
         connstr += '&';
     }
+    if (o_keypath.passed()) {
+        connstr += "keypath=";
+        connstr += o_keypath.result();
+        connstr += '&';
+    }
     if (o_ssl.passed()) {
         connstr += "ssl=";
         connstr += o_ssl.result();
@@ -417,6 +433,17 @@ ConnParams::doCtls(lcb_t instance)
 
         // Set the detailed error codes option
         doSctl<int>(instance, LCB_CNTL_DETAILED_ERRCODES, 1);
+
+        {
+            int opts = LCB_COMPRESS_IN;
+            if (o_compress.passed()) {
+                opts |= LCB_COMPRESS_OUT;
+                if (o_compress.numSpecified() > 1) {
+                    opts |= LCB_COMPRESS_FORCE;
+                }
+            }
+            doPctl(instance, LCB_CNTL_COMPRESSION_OPTS, &opts);
+        }
     } catch (lcb_error_t &err) {
         return err;
     }