libcouchbase 0.3.3 → 1.0.0

data/ext/libcouchbase/tests/vbucket/confdata/memd_ketama_config.json

@@ -0,0 +1,31 @@
+{
+   "bucketCapabilitiesVer" : "",
+   "name" : "memd",
+   "nodeLocator" : "ketama",
+   "nodes" : [
+      {
+         "ports" : {
+            "direct" : 11210
+         },
+         "hostname" : "192.168.1.103:8091"
+      },
+      {
+         "hostname" : "192.168.1.101:8091",
+         "ports" : {
+            "direct" : 11210
+         }
+      },
+      {
+         "hostname" : "192.168.1.102:8091",
+         "ports" : {
+            "direct" : 11210
+         }
+      },
+      {
+         "hostname" : "$HOST:8091",
+         "ports" : {
+            "direct" : 11210
+         }
+      }
+   ]
+}

data/ext/libcouchbase/tests/vbucket/t_config.cc

@@ -55,10 +55,10 @@ ConfigTest::testConfig(const char *fname, bool checkNew)
     ASSERT_TRUE(vbc != NULL);
     int rv = lcbvb_load_json(vbc, testData.c_str());
     ASSERT_EQ(0, rv);
-    ASSERT_GT(vbc->nsrv, 0);
+    ASSERT_GT(vbc->nsrv, (unsigned int)0);
 
     if (vbc->dtype == LCBVB_DIST_VBUCKET) {
-        ASSERT_GT(vbc->nvb, 0);
+        ASSERT_GT(vbc->nvb, (unsigned int)0);
 
         for (unsigned ii = 0; ii < vbc->nvb; ii++) {
             lcbvb_vbmaster(vbc, ii);
@@ -284,7 +284,7 @@ TEST_F(ConfigTest, testNondataNodes)
 
         lcbvb_map_key(cfg_old, s.c_str(), s.size(), &vbid, &ix_exp);
         lcbvb_map_key(cfg_ex, s.c_str(), s.size(), &vbid, &ix_cur);
-        ASSERT_TRUE(ix_exp > -1 && ix_exp <  cfg_ex->ndatasrv);
+        ASSERT_TRUE(ix_exp > -1 && ix_exp < (int)cfg_ex->ndatasrv);
         ASSERT_EQ(ix_exp, ix_cur);
     }
 
@@ -298,7 +298,7 @@ TEST_F(ConfigTest, testNondataNodes)
             if (newix == -1) {
                 continue;
             } else {
-                ASSERT_TRUE(newix < cfg_ex->ndatasrv);
+                ASSERT_TRUE(newix < (int)cfg_ex->ndatasrv);
             }
         }
     }
@@ -310,7 +310,7 @@ TEST_F(ConfigTest, testNondataNodes)
         const string& s = keys[ii];
         lcbvb_map_key(cfg_old, s.c_str(), s.size(), &vbid, &ix_exp);
         lcbvb_map_key(cfg_ex, s.c_str(), s.size(), &vbid, &ix_cur);
-        ASSERT_TRUE(ix_exp > -1 && ix_exp < cfg_old->ndatasrv);
+        ASSERT_TRUE(ix_exp > -1 && ix_exp < (int)cfg_old->ndatasrv);
         ASSERT_EQ(ix_exp, ix_cur);
     }
 
@@ -339,3 +339,33 @@ TEST_F(ConfigTest, testKetamaUniformity)
     ASSERT_STREQ("localhost:12006", vbc->servers[3].authority);
     lcbvb_destroy(vbc);
 }
+
+TEST_F(ConfigTest, testKetamaCompliance) {
+    string txt = getConfigFile("memd_ketama_config.json");
+    lcbvb_CONFIG *vbc = lcbvb_parse_json(txt.c_str());
+    ASSERT_TRUE(vbc != NULL);
+    ASSERT_EQ(4, vbc->nsrv);
+    ASSERT_EQ(LCBVB_DIST_KETAMA, vbc->dtype);
+
+    lcbvb_replace_host(vbc, "192.168.1.104");
+    // Now, load the hash file
+    string expected_txt = getConfigFile("ketama_expected.json");
+    Json::Value json;
+    ASSERT_TRUE(Json::Reader().parse(expected_txt, json));
+
+    ASSERT_EQ(json.size(), vbc->ncontinuum);
+
+    // Iterate over the continuum in the vbuckets
+    for (size_t ii = 0; ii < json.size(); ++ii) {
+        const Json::Value& cur = json[static_cast<int>(ii)];
+        unsigned exp_hash = cur["hash"].asUInt();
+        string exp_server = cur["hostname"].asString();
+        unsigned got_hash = vbc->continuum[ii].point;
+        unsigned got_index = vbc->continuum[ii].index;
+
+        ASSERT_EQ(exp_server, vbc->servers[got_index].authority);
+        ASSERT_EQ(exp_hash, got_hash);
+    }
+
+    lcbvb_destroy(vbc);
+}

data/ext/libcouchbase/tools/CMakeLists.txt

@@ -4,7 +4,7 @@ FILE(GLOB T_COMMONSRC common/*.cc)
 ADD_LIBRARY(lcbtools OBJECT ${T_COMMONSRC})
 
 ADD_EXECUTABLE(cbc cbc.cc
-    $<TARGET_OBJECTS:lcbtools> $<TARGET_OBJECTS:cliopts>)
+    $<TARGET_OBJECTS:lcbtools> $<TARGET_OBJECTS:cliopts> $<TARGET_OBJECTS:lcb_jsoncpp>)
 TARGET_LINK_LIBRARIES(cbc couchbase)
 
 ADD_EXECUTABLE(cbc-pillowfight cbc-pillowfight.cc
@@ -38,7 +38,7 @@ IF(NOT WIN32)
         cat create observe observe-seqno incr decr mcflush hash lock
         unlock rm stats version verbosity view n1ql admin
         bucket-create bucket-delete bucket-flush connstr write-config strerror
-        touch)
+        touch role-list user-list user-upsert user-delete)
 
     FOREACH(subcmd IN ITEMS ${CBC_SUBCOMMANDS})
         ADD_CUSTOM_COMMAND(TARGET cbc POST_BUILD

data/ext/libcouchbase/tools/cbc-handlers.h

@@ -367,6 +367,134 @@ protected:
 
 };
 
+class RbacHandler : public AdminHandler {
+public:
+    HANDLER_USAGE("[OPTIONS ...]")
+    RbacHandler(const char *name) : AdminHandler(name),
+        o_raw('r', "raw")
+    {
+        o_raw.description("Do not reformat output from server (display JSON response)");
+    }
+
+protected:
+    virtual void run();
+    virtual void format() = 0;
+    virtual void addOptions() {
+        AdminHandler::addOptions();
+        parser.addOption(o_raw);
+    }
+
+private:
+    cliopts::BoolOption o_raw;
+};
+
+class RoleListHandler : public RbacHandler {
+public:
+    HANDLER_DESCRIPTION("List roles")
+        RoleListHandler() : RbacHandler("role-list")
+    {
+    }
+
+protected:
+    virtual void format();
+    virtual void addOptions() {
+        RbacHandler::addOptions();
+    }
+    std::string getURI() { return "/settings/rbac/roles"; }
+    const std::string& getBody() { static std::string e; return e; }
+    lcb_http_method_t getMethod() { return LCB_HTTP_METHOD_GET; }
+};
+
+class UserListHandler : public RbacHandler {
+public:
+    HANDLER_DESCRIPTION("List users")
+        UserListHandler() : RbacHandler("user-list")
+    {
+    }
+
+protected:
+    virtual void format();
+    virtual void addOptions() {
+        RbacHandler::addOptions();
+    }
+    std::string getURI() { return "/settings/rbac/users"; }
+    const std::string& getBody() { static std::string e; return e; }
+    lcb_http_method_t getMethod() { return LCB_HTTP_METHOD_GET; }
+};
+
+class UserDeleteHandler : public AdminHandler {
+public:
+    HANDLER_DESCRIPTION("Delete a user")
+    HANDLER_USAGE("NAME [OPTIONS ...]")
+    UserDeleteHandler() : AdminHandler("user-delete"),
+        o_domain("domain")
+    {
+        o_domain.description("The domain, where user account defined {local,external}").setDefault("local");
+    }
+
+protected:
+    virtual void addOptions() {
+        AdminHandler::addOptions();
+        parser.addOption(o_domain);
+    }
+    void run() {
+        name = getRequiredArg();
+        domain = o_domain.result();
+        if (domain != "local" && domain != "external") {
+            throw BadArg("Unrecognized domain type");
+        }
+        AdminHandler::run();
+    }
+    std::string getURI() { return std::string("/settings/rbac/users/") + domain + "/" + name; }
+    const std::string& getBody() { static std::string e; return e; }
+    lcb_http_method_t getMethod() { return LCB_HTTP_METHOD_DELETE; }
+
+private:
+    cliopts::StringOption o_domain;
+    std::string name;
+    std::string domain;
+};
+
+class UserUpsertHandler : public AdminHandler {
+public:
+    HANDLER_DESCRIPTION("Create or update a user")
+    HANDLER_USAGE("NAME [OPTIONS ...]")
+    UserUpsertHandler() : AdminHandler("user-upsert"),
+        o_domain("domain"),
+        o_full_name("full-name"),
+        o_password("user-password"),
+        o_roles("role")
+    {
+        o_domain.description("The domain, where user account defined {local,external}").setDefault("local");
+        o_full_name.description("The user's fullname");
+        o_roles.description("The role associated with user (can be specified multiple times if needed)");
+        o_password.description("The password for the user");
+    }
+
+protected:
+    virtual void addOptions() {
+        AdminHandler::addOptions();
+        parser.addOption(o_domain);
+        parser.addOption(o_full_name);
+        parser.addOption(o_roles);
+        parser.addOption(o_password);
+    }
+    virtual void run();
+    std::string getURI() { return std::string("/settings/rbac/users/") + domain + "/" + name; }
+    const std::string& getBody() { return body; }
+    std::string getContentType() { return "application/x-www-form-urlencoded"; }
+    lcb_http_method_t getMethod() { return LCB_HTTP_METHOD_PUT; }
+
+private:
+    cliopts::StringOption o_domain;
+    cliopts::StringOption o_full_name;
+    cliopts::StringOption o_password;
+    cliopts::ListOption o_roles;
+    std::string name;
+    std::string domain;
+    std::string body;
+};
+
 class BucketCreateHandler : public AdminHandler {
 public:
     HANDLER_DESCRIPTION("Create a bucket")

data/ext/libcouchbase/tools/cbc-n1qlback.cc

@@ -74,6 +74,8 @@ public:
         start_time = last_update;
     }
 
+    size_t nerrors() { return n_errors; }
+
     void update_row(size_t n = 1) { n_rows += n; update_display(); }
     void update_done(size_t n = 1) { n_queries += n; update_display(); }
     void update_error(size_t n = 1) { n_errors += n; update_display(); }
@@ -139,8 +141,8 @@ private:
             final_suffix = "\n";
         }
 
-        printf("%sQUERIES/SEC: %lu\n", prefix, n_queries / duration);
-        printf("%sROWS/SEC:    %lu\n", prefix, n_rows / duration);
+        printf("%sQUERIES/SEC: %lu\n", prefix, (long int)(n_queries / duration));
+        printf("%sROWS/SEC:    %lu\n", prefix, (long int)(n_rows / duration));
         printf("%sERRORS:      %lu%s", prefix, n_errors, final_suffix);
 
         if (hg != NULL) {
@@ -168,7 +170,7 @@ Metrics GlobalMetrics;
 class Configuration
 {
 public:
-    Configuration() : o_file("queryfile"), o_threads("num-threads") {
+    Configuration() : o_file("queryfile"), o_threads("num-threads"), o_errlog("error-log"), m_errlog(NULL) {
         o_file.mandatory(true);
         o_file.description(
             "Path to a file containing all the queries to execute. "
@@ -178,12 +180,24 @@ public:
         o_threads.description("Number of threads to run");
         o_threads.abbrev('t');
         o_threads.setDefault(1);
+
+        o_errlog.description(
+            "Path to a file containing failed queries");
+        o_errlog.abbrev('e');
+        o_errlog.setDefault("");
     }
 
+    ~Configuration() {
+        if (m_errlog != NULL) {
+            delete m_errlog;
+            m_errlog = NULL;
+        }
+    }
     void addToParser(Parser& parser)
     {
         parser.addOption(o_file);
         parser.addOption(o_threads);
+        parser.addOption(o_errlog);
         m_params.addToParser(parser);
     }
 
@@ -200,22 +214,38 @@ public:
 
         string curline;
         while (std::getline(ifs, curline).good() && !ifs.eof()) {
-            m_queries.push_back(curline);
+            if (!curline.empty()) {
+                m_queries.push_back(curline);
+            }
         }
         if (m_params.useTimings()) {
             GlobalMetrics.prepare_timings();
         }
+
+        if (o_errlog.passed()) {
+            m_errlog = new std::ofstream(o_errlog.const_result().c_str());
+            if (!m_errlog->is_open()) {
+                int ec_save = errno;
+                string errstr(o_file.const_result());
+                errstr += ": ";
+                errstr += strerror(ec_save);
+                throw std::runtime_error(errstr);
+            }
+        }
     }
 
     void set_cropts(lcb_create_st &opts) { m_params.fillCropts(opts); }
     const vector<string>& queries() const { return m_queries; }
     size_t nthreads() { return o_threads.result(); }
+    std::ofstream* errlog() { return m_errlog; }
 
 private:
     vector<string> m_queries;
     StringOption o_file;
     UIntOption o_threads;
     ConnParams m_params;
+    StringOption o_errlog;
+    std::ofstream *m_errlog;
 };
 
 extern "C" { static void n1qlcb(lcb_t, int, const lcb_RESPN1QL *resp); }
@@ -287,16 +317,25 @@ public:
 
         if (resp->rflags & LCB_RESP_F_FINAL) {
             if (resp->rc != LCB_SUCCESS) {
-                log_error(resp->rc);
+                if (m_errlog != NULL) {
+                    std::stringstream ss;
+                    ss.write(m_cmd.query, m_cmd.nquery);
+                    ss << endl;
+                    ss.write(resp->row, resp->nrow);
+                    log_error(resp->rc, ss.str().c_str(), ss.str().size());
+                } else {
+                    log_error(resp->rc, NULL, 0);
+                }
             }
         } else {
             last_nrow++;
         }
     }
 
-    ThreadContext(lcb_t instance, const vector<string>& initial_queries)
+    ThreadContext(lcb_t instance, const vector<string>& initial_queries, std::ofstream *errlog)
     : m_instance(instance), last_nerr(0), last_nrow(0),
-      m_metrics(&GlobalMetrics), m_cancelled(false), m_thr(NULL)
+      m_metrics(&GlobalMetrics), m_cancelled(false), m_thr(NULL),
+      m_errlog(errlog)
     {
         memset(&m_cmd, 0, sizeof m_cmd);
         m_cmd.content_type = "application/json";
@@ -309,11 +348,25 @@ public:
 
 private:
 
-    void log_error(lcb_error_t)
+    void log_error(lcb_error_t err, const char* info, size_t ninfo)
     {
+        size_t erridx;
         m_metrics->lock();
         m_metrics->update_error();
+        erridx = m_metrics->nerrors();
         m_metrics->unlock();
+
+        if (m_errlog != NULL) {
+            std::stringstream ss;
+            ss << "[" << erridx << "] 0x" << std::hex << err << ", "
+               << lcb_strerror(NULL, err) << endl;
+            if (ninfo) {
+                ss.write(info, ninfo);
+                ss << endl;
+            }
+            *m_errlog << ss.str();
+            m_errlog->flush();
+        }
     }
 
     void run_one_query(const string& txt)
@@ -330,7 +383,7 @@ private:
 
         lcb_error_t rc = lcb_n1ql_query(m_instance, &qctx, &m_cmd);
         if (rc != LCB_SUCCESS) {
-            log_error(rc);
+            log_error(rc, txt.c_str(), txt.size());
         } else {
             lcb_wait(m_instance);
             m_metrics->lock();
@@ -352,6 +405,7 @@ private:
     #else
     void *m_thr;
     #endif
+    std::ofstream *m_errlog;
 };
 
 static void n1qlcb(lcb_t, int, const lcb_RESPN1QL *resp)
@@ -409,7 +463,7 @@ static void real_main(int argc, char **argv) {
             throw std::runtime_error("Cluster does not support N1QL!");
         }
 
-        ThreadContext* cx = new ThreadContext(instance, config.queries());
+        ThreadContext* cx = new ThreadContext(instance, config.queries(), config.errlog());
         threads.push_back(cx);
         instances.push_back(instance);
     }

data/ext/libcouchbase/tools/cbc-pillowfight.cc

@@ -170,7 +170,7 @@ public:
             }
             maxCycles = est;
             o_sequential.setDefault(true);
-            fprintf(stderr, "Populating using %lu cycles\n", maxCycles);
+            fprintf(stderr, "Populating using %d cycles\n", maxCycles);
         }
 
         if (depr.iterations.passed()) {
@@ -630,7 +630,7 @@ private:
 
         const lcb_U64 elapsed_ns = now - previous_time;
         const lcb_U64 wanted_duration_ns =
-                config.opsPerCycle * 1e9 / config.getRateLimit();
+                (config.getNumThreads() * config.opsPerCycle * 1e9) / config.getRateLimit();
         // On first invocation no previous_time, so skip attempting to sleep.
         if (elapsed_ns > 0 && elapsed_ns < wanted_duration_ns) {
             // Dampen the sleep time by averaging with the previous