libcouchbase 1.2.8 → 1.3.0

data/ext/libcouchbase/tests/socktests/socktest.cc

@@ -206,7 +206,7 @@ Loop::initSockCommon(ESocket *sock)
 void
 Loop::connectPooled(ESocket *sock, lcb_host_t *host, unsigned mstmo)
 {
-    lcb_host_t tmphost;
+    lcb_host_t tmphost = {0};
     sock->parent = this;
     if (!host) {
         populateHost(&tmphost);
@@ -222,7 +222,7 @@ Loop::connectPooled(ESocket *sock, lcb_host_t *host, unsigned mstmo)
 void
 Loop::connect(ESocket *sock, lcb_host_t *host, unsigned mstmo)
 {
-    lcb_host_t tmphost;
+    lcb_host_t tmphost = {0};
 
     if (host == NULL) {
         populateHost(&tmphost);

data/ext/libcouchbase/tests/socktests/t_basic.cc

@@ -62,7 +62,7 @@ static bool isRefused(lcbio_OSERR err)
 TEST_F(SockConnTest, testRefused)
 {
     ESocket sock;
-    lcb_host_t host;
+    lcb_host_t host = {0};
     strcpy(host.host, "localhost");
     strcpy(host.port, "1");
     loop->connect(&sock, &host, 100000);
@@ -73,7 +73,7 @@ TEST_F(SockConnTest, testRefused)
 TEST_F(SockConnTest, testBadDomain)
 {
     ESocket sock;
-    lcb_host_t host;
+    lcb_host_t host = {0};
     strcpy(host.host, "domain-should-not-work.nonexist.com");
     strcpy(host.port, "123");
     loop->connect(&sock, &host, 1000);
@@ -83,7 +83,7 @@ TEST_F(SockConnTest, testBadDomain)
 TEST_F(SockConnTest, testInvalidPort)
 {
     ESocket sock;
-    lcb_host_t host;
+    lcb_host_t host = {0};
     strcpy(host.host, "localhost");
     strcpy(host.port, "111111111");
     loop->connect(&sock, &host, 1000);
@@ -93,7 +93,7 @@ TEST_F(SockConnTest, testInvalidPort)
 TEST_F(SockTest, testEmptyHost)
 {
     ESocket sock;
-    lcb_host_t host;
+    lcb_host_t host = {0};
     host.host[0] = '\0';
     host.port[0] = '\0';
     loop->connect(&sock, &host, 1000);
@@ -103,7 +103,7 @@ TEST_F(SockTest, testEmptyHost)
 TEST_F(SockConnTest, testCancellation)
 {
     ESocket sock;
-    lcb_host_t host;
+    lcb_host_t host = {0};
     loop->populateHost(&host);
     sock.creq = lcbio_connect(
             loop->iot, loop->settings, &host, 100000, NULL, NULL);
@@ -132,7 +132,7 @@ conncb_1(lcbio_SOCKET *sock, void *arg, lcb_error_t err, lcbio_OSERR syserr)
 TEST_F(SockConnTest, testImmediateUnref)
 {
     ESocket sock;
-    lcb_host_t host;
+    lcb_host_t host = {0};
     sock.parent = loop;
     loop->populateHost(&host);
     sock.creq = lcbio_connect(

data/ext/libcouchbase/tests/socktests/t_manager.cc

@@ -30,7 +30,7 @@ TEST_F(SockMgrTest, testBasic)
 
 TEST_F(SockMgrTest, testCancellation)
 {
-    lcb_host_t host;
+    lcb_host_t host = {0};
     loop->populateHost(&host);
     lcb::io::ConnectionRequest *req = loop->sockpool->get(host, LCB_MS2US(1000), NULL, NULL);
     ASSERT_FALSE(req == NULL);

data/ext/libcouchbase/tests/socktests/t_ssl.cc

@@ -16,7 +16,7 @@ protected:
 
         SockTest::SetUp();
         loop->settings->sslopts = LCB_SSL_ENABLED|LCB_SSL_NOVERIFY;
-        loop->settings->ssl_ctx = lcbio_ssl_new(NULL, 1, &errp, loop->settings);
+        loop->settings->ssl_ctx = lcbio_ssl_new(NULL, NULL, NULL, 1, &errp, loop->settings);
         loop->server->factory = TestServer::sslSocketFactory;
         EXPECT_FALSE(loop->settings->ssl_ctx == NULL) << lcb_strerror(NULL, errp);
     }

data/ext/libcouchbase/tools/CMakeLists.txt

@@ -62,7 +62,7 @@ IF(NOT WIN32)
         cat create observe observe-seqno incr decr mcflush hash lock
         unlock rm stats version verbosity view n1ql admin ping
         bucket-create bucket-delete bucket-flush connstr write-config strerror
-        touch role-list user-list user-upsert user-delete)
+        touch role-list user-list user-upsert user-delete watch)
 
     FOREACH(subcmd IN ITEMS ${CBC_SUBCOMMANDS})
         ADD_CUSTOM_COMMAND(TARGET cbc POST_BUILD

data/ext/libcouchbase/tools/cbc-handlers.h

@@ -194,6 +194,23 @@ private:
     cliopts::BoolOption o_keystats;
 };
 
+class WatchHandler : public Handler {
+public:
+    HANDLER_DESCRIPTION("Aggregate and display server statistics")
+    HANDLER_USAGE("[KEYS ....] [OPTIONS ...]")
+        WatchHandler() : Handler("watch"), o_interval("interval") {
+        o_interval.abbrev('n').description("Update interval in seconds").setDefault(1);
+    }
+protected:
+    void run();
+    void addOptions() {
+        Handler::addOptions();
+        parser.addOption(o_interval);
+    }
+private:
+    cliopts::UIntOption o_interval;
+};
+
 class VerbosityHandler : public Handler {
 public:
     HANDLER_DESCRIPTION("Modify the memcached logging level")

data/ext/libcouchbase/tools/cbc-n1qlback.cc

@@ -141,9 +141,9 @@ private:
             final_suffix = "\n";
         }
 
-        printf("%sQUERIES/SEC: %lu\n", prefix, (long int)(n_queries / duration));
-        printf("%sROWS/SEC:    %lu\n", prefix, (long int)(n_rows / duration));
-        printf("%sERRORS:      %lu%s", prefix, n_errors, final_suffix);
+        printf("%sQUERIES/SEC: %lu\n", prefix, (unsigned long)(n_queries / duration));
+        printf("%sROWS/SEC:    %lu\n", prefix, (unsigned long)(n_rows / duration));
+        printf("%sERRORS:      %lu%s", prefix, (unsigned long)n_errors, final_suffix);
 
         if (hg != NULL) {
             hg->write();
@@ -213,11 +213,14 @@ public:
         }
 
         string curline;
-        while (std::getline(ifs, curline).good() && !ifs.eof()) {
+        while (ifs.good()) {
+            std::getline(ifs, curline);
             if (!curline.empty()) {
                 m_queries.push_back(curline);
             }
         }
+        std::cerr << "Loaded " << m_queries.size() << " queries "
+                  << "from \"" << o_file.const_result() << "\"" << std::endl;
         if (m_params.useTimings()) {
             GlobalMetrics.prepare_timings();
         }

data/ext/libcouchbase/tools/cbc-pillowfight.cc

@@ -32,6 +32,7 @@
 #include <signal.h>
 #ifndef WIN32
 #include <pthread.h>
+#include <libcouchbase/metrics.h>
 #else
 #define usleep(n) Sleep(n/1000)
 #endif
@@ -105,6 +106,7 @@ public:
         o_keyPrefix("key-prefix"),
         o_numThreads("num-threads"),
         o_randSeed("random-seed"),
+        o_randomBody("random-body"),
         o_setPercent("set-pct"),
         o_minSize("min-size"),
         o_maxSize("max-size"),
@@ -121,13 +123,19 @@ public:
         o_noop("noop"),
         o_sdPathCount("pathcount"),
         o_populateOnly("populate-only"),
-        o_exptime("expiry")
+        o_exptime("expiry"),
+        o_collection("collection"),
+        o_separator("separator"),
+        o_persist("persist-to"),
+        o_replicate("replicate-to"),
+        o_lock("lock")
     {
         o_multiSize.setDefault(100).abbrev('B').description("Number of operations to batch");
         o_numItems.setDefault(1000).abbrev('I').description("Number of items to operate on");
         o_keyPrefix.abbrev('p').description("key prefix to use");
         o_numThreads.setDefault(1).abbrev('t').description("The number of threads to use");
         o_randSeed.setDefault(0).abbrev('s').description("Specify random seed").hide();
+        o_randomBody.setDefault(false).abbrev('R').description("Randomize document body (otherwise use 'x' and '*' to fill)");
         o_setPercent.setDefault(33).abbrev('r').description("The percentage of operations which should be mutations");
         o_minSize.setDefault(50).abbrev('m').description("Set minimum payload size");
         o_maxSize.setDefault(5120).abbrev('M').description("Set maximum payload size");
@@ -138,7 +146,7 @@ public:
         o_startAt.setDefault(0).description("For sequential access, set the first item");
         o_rateLimit.setDefault(0).description("Set operations per second limit (per thread)");
         o_userdocs.description("User documents to load (overrides --min-size and --max-size");
-        o_writeJson.description("Enable writing JSON values (rather than bytes)");
+        o_writeJson.abbrev('J').description("Enable writing JSON values (rather than bytes)");
         o_templatePairs.description("Values for templates to be inserted into user documents");
         o_templatePairs.argdesc("FIELD,MIN,MAX[,SEQUENTIAL]").hide();
         o_subdoc.description("Use subdoc instead of fulldoc operations");
@@ -146,6 +154,12 @@ public:
         o_sdPathCount.description("Number of subdoc paths per command").setDefault(1);
         o_populateOnly.description("Exit after documents have been populated");
         o_exptime.description("Set TTL for items").abbrev('e');
+        o_collection.description("Allowed collection name (could be specified multiple times)").hide();
+        o_separator.setDefault(":").description("Separator for collection prefix in keys").hide();
+        o_persist.description("Wait until item is persisted to this number of nodes (-1 for master+replicas)").setDefault(0);
+        o_replicate.description("Wait until item is replicated to this number of nodes (-1 for all replicas)").setDefault(0);
+        o_lock.description("Lock keys for updates for given time (will not lock when set to zero)").setDefault(0);
+        params.getTimings().description("Enable command timings (second time to dump timings automatically)");
     }
 
     void processOptions() {
@@ -153,7 +167,18 @@ public:
         prefix = o_keyPrefix.result();
         setprc = o_setPercent.result();
         shouldPopulate = !o_noPopulate.result();
+        persistTo = o_persist.result();
+        replicateTo = o_replicate.result();
+        lockTime = o_lock.result();
+        if (lockTime && o_numItems < opsPerCycle * o_numThreads) {
+            fprintf(stderr, "The --num-items=%d cannot be smaller than --batch-size=%d multiplied to --num-thread=%d when used with --lock=%d\n",
+                    (int)o_numItems, (int)opsPerCycle, (int)o_numThreads, (int)lockTime);
+            exit(EXIT_FAILURE);
+        }
 
+        if (o_keyPrefix.passed() && o_collection.passed()) {
+            throw std::runtime_error("The --collection is not compatible with --key-prefix");
+        }
         if (depr.loop.passed()) {
             fprintf(stderr, "The --loop/-l option is deprecated. Use --num-cycles\n");
             maxCycles = -1;
@@ -211,17 +236,17 @@ public:
 
         if (specs.empty()) {
             if (o_writeJson.result()) {
-                docgen = new JsonDocGenerator(o_minSize.result(), o_maxSize.result());
+                docgen = new JsonDocGenerator(o_minSize.result(), o_maxSize.result(), o_randomBody.numSpecified());
             } else if (!userdocs.empty()) {
                 docgen = new PresetDocGenerator(userdocs);
             } else {
-                docgen = new RawDocGenerator(o_minSize.result(), o_maxSize.result());
+                docgen = new RawDocGenerator(o_minSize.result(), o_maxSize.result(), o_randomBody.numSpecified());
             }
         } else {
             if (o_writeJson.result()) {
                 if (userdocs.empty()) {
                     docgen = new PlaceholderJsonGenerator(
-                        o_minSize.result(), o_maxSize.result(), specs);
+                        o_minSize.result(), o_maxSize.result(), specs, o_randomBody.numSpecified());
                 } else {
                     docgen = new PlaceholderJsonGenerator(userdocs, specs);
                 }
@@ -237,6 +262,17 @@ public:
         if (o_sdPathCount.passed()) {
             o_subdoc.setDefault(true);
         }
+
+        if (o_collection.passed()) {
+            string separator = o_separator.result();
+            if (separator.empty()) {
+                throw std::runtime_error("Collection name separator must not be empty");
+            }
+            vector<string> names = o_collection.result();
+            for (size_t ii = 0; ii < names.size(); ii++) {
+                collections.push_back(names[ii] + separator);
+            }
+        }
     }
 
     void addOptions(Parser& parser) {
@@ -245,6 +281,7 @@ public:
         parser.addOption(o_keyPrefix);
         parser.addOption(o_numThreads);
         parser.addOption(o_randSeed);
+        parser.addOption(o_randomBody);
         parser.addOption(o_setPercent);
         parser.addOption(o_noPopulate);
         parser.addOption(o_minSize);
@@ -262,11 +299,16 @@ public:
         parser.addOption(o_sdPathCount);
         parser.addOption(o_populateOnly);
         parser.addOption(o_exptime);
+        parser.addOption(o_collection);
+        parser.addOption(o_separator);
+        parser.addOption(o_persist);
+        parser.addOption(o_replicate);
+        parser.addOption(o_lock);
         params.addToParser(parser);
         depr.addOptions(parser);
     }
 
-    bool isTimings(void) { return params.useTimings(); }
+    int numTimings(void) { return params.numTimings(); }
 
     bool isLoopDone(size_t niter) {
         if (maxCycles == -1) {
@@ -282,6 +324,8 @@ public:
     bool sequentialAccess() { return o_sequential; }
     bool isSubdoc() { return o_subdoc; }
     bool isNoop() { return o_noop.result(); }
+    bool useCollections() { return o_collection.passed(); }
+    bool writeJson() { return o_writeJson.result(); }
     unsigned firstKeyOffset() { return o_startAt; }
     uint32_t getNumItems() { return o_numItems; }
     uint32_t getRateLimit() { return o_rateLimit; }
@@ -296,6 +340,10 @@ public:
     bool hasTemplates;
     ConnParams params;
     const DocGeneratorBase *docgen;
+    vector<string> collections;
+    int replicateTo;
+    int persistTo;
+    int lockTime;
 
 private:
     UIntOption o_multiSize;
@@ -303,6 +351,7 @@ private:
     StringOption o_keyPrefix;
     UIntOption o_numThreads;
     UIntOption o_randSeed;
+    BoolOption o_randomBody;
     UIntOption o_setPercent;
     UIntOption o_minSize;
     UIntOption o_maxSize;
@@ -331,6 +380,12 @@ private:
 
     UIntOption o_exptime;
 
+    ListOption o_collection;
+    StringOption o_separator;
+    IntOption o_persist;
+    IntOption o_replicate;
+
+    IntOption o_lock;
     DeprecatedOptions depr;
 } config;
 
@@ -341,7 +396,7 @@ void log(const char *format, ...)
 
     va_start(args, format);
     vsprintf(buffer, format, args);
-    if (config.isTimings()) {
+    if (config.numTimings() > 0) {
         std::cerr << "[" << std::fixed << lcb_nstime() / 1000000000.0 << "] ";
     }
     std::cerr << buffer << std::endl;
@@ -352,16 +407,24 @@ void log(const char *format, ...)
 
 extern "C" {
 static void operationCallback(lcb_t, int, const lcb_RESPBASE*);
+static void storeCallback(lcb_t, int, const lcb_RESPBASE *);
 }
 
+class ThreadContext;
+
 class InstanceCookie {
 public:
     InstanceCookie(lcb_t instance) {
         lcb_set_cookie(instance, this);
         lastPrint = 0;
-        if (config.isTimings()) {
+        if (config.numTimings() > 0) {
             hg.install(instance, stdout);
         }
+        stats.total = 0;
+        stats.retried = 0;
+        stats.etmpfail = 0;
+        stats.eexist = 0;
+        stats.etimeout = 0;
     }
 
     static InstanceCookie* get(lcb_t instance) {
@@ -369,7 +432,7 @@ public:
     }
 
 
-    static void dumpTimings(lcb_t instance, const char *header, bool force=false) {
+    static void dumpTimings(lcb_t instance, const char *header = NULL, bool force=false) {
         time_t now = time(NULL);
         InstanceCookie *ic = get(instance);
 
@@ -380,19 +443,37 @@ public:
         }
 
         Histogram &h = ic->hg;
-        printf("[%f %s]\n", lcb_nstime() / 1000000000.0, header);
-        printf("              +---------+---------+---------+---------+\n");
+        if (header) {
+            printf("[%f %s]\n", lcb_nstime() / 1000000000.0, header);
+        }
+        printf("                +---------+---------+---------+---------+\n");
         h.write();
-        printf("              +----------------------------------------\n");
+        printf("                +----------------------------------------\n");
     }
 
+    void setContext(ThreadContext *context) {
+        m_context = context;
+    }
+
+    ThreadContext * getContext() {
+        return m_context;
+    }
+
+    struct {
+        size_t total;
+        size_t retried;
+        size_t etmpfail;
+        size_t eexist;
+        size_t etimeout;
+    } stats;
 private:
     time_t lastPrint;
     Histogram hg;
+    ThreadContext *m_context;
 };
 
 struct NextOp {
-    NextOp() : m_seqno(0), m_mode(GET) {}
+    NextOp() : m_seqno(0), m_mode(GET), m_cas(0) {}
 
     string m_key;
     uint32_t m_seqno;
@@ -401,6 +482,7 @@ struct NextOp {
     // The mode here is for future use with subdoc
     enum Mode { STORE, GET, SDSTORE, SDGET, NOOP };
     Mode m_mode;
+    uint64_t m_cas;
 };
 
 class OpGenerator {
@@ -410,7 +492,9 @@ public:
     virtual ~OpGenerator() {};
     virtual void setNextOp(NextOp& op) = 0;
     virtual void setValue(NextOp& op) = 0;
+    virtual void populateIov(uint32_t, vector<lcb_IOV>&) = 0;
     virtual bool inPopulation() const = 0;
+    virtual void checkin(uint32_t) = 0;
     virtual const char *getStageString() const = 0;
 
 protected:
@@ -427,11 +511,14 @@ public:
     }
 
     void setValue(NextOp&) {}
+    void populateIov(uint32_t, vector<lcb_IOV>&) {}
 
     bool inPopulation() const {
         return false;
     }
 
+    void checkin(uint32_t) {}
+
     const char *getStageString() const {
         return "Run";
     }
@@ -481,6 +568,10 @@ public:
         m_local_genstate->populateIov(op.m_seqno, op.m_valuefrags);
     }
 
+    void populateIov(uint32_t seq, vector<lcb_IOV>& iov_out) {
+        m_local_genstate->populateIov(seq, iov_out);
+    }
+
     void setNextOp(NextOp& op) {
         bool store_override = false;
 
@@ -494,10 +585,10 @@ public:
             }
         }
 
-        if (m_force_sequential) {
-            op.m_seqno = m_gensequence->next();
+        if (m_in_population || !config.lockTime) {
+            op.m_seqno = (m_force_sequential ? m_gensequence : m_genrandom)->next();
         } else {
-            op.m_seqno = m_genrandom->next();
+            op.m_seqno = (m_force_sequential ? m_gensequence : m_genrandom)->checkout();
         }
 
         if (store_override) {
@@ -531,6 +622,10 @@ public:
         return m_in_population;
     }
 
+    void checkin(uint32_t seqno) {
+        (m_force_sequential ? m_gensequence : m_genrandom)->checkin(seqno);
+    }
+
     const char *getStageString() const {
         if (m_in_population) {
             return "Populate";
@@ -554,14 +649,16 @@ private:
         uint32_t seqno = op.m_seqno;
         char buffer[21];
         snprintf(buffer, sizeof(buffer), "%020d", seqno);
-        op.m_key.assign(config.getKeyPrefix() + buffer);
+        string &prefix = config.useCollections()
+            ? config.collections[seqno % config.collections.size()]
+            : config.getKeyPrefix();
+        op.m_key.assign(prefix + buffer);
     }
 
 
     SeqGenerator *m_genrandom;
     SeqGenerator *m_gensequence;
     size_t m_gencount;
-    int m_id;
 
     bool m_force_sequential;
     bool m_in_population;
@@ -571,6 +668,8 @@ private:
     SubdocGeneratorState *m_sdgenstate;
 };
 
+#define OPFLAGS_LOCKED 0x01
+
 class ThreadContext
 {
 public:
@@ -591,79 +690,52 @@ public:
         return gen && (gen->inPopulation() || !retryq.empty());
     }
 
+    void checkin(uint32_t seqno) {
+        if (gen) {
+            gen->checkin(seqno);
+        }
+    }
+
     void singleLoop() {
         bool hasItems = false;
-        NextOp opinfo;
-        unsigned exptime = config.getExptime();
 
         lcb_sched_enter(instance);
         for (size_t ii = 0; ii < config.opsPerCycle; ++ii) {
-            gen->setNextOp(opinfo);
-
-            switch (opinfo.m_mode) {
-            case NextOp::STORE: {
-                lcb_CMDSTORE scmd = { 0 };
-                scmd.operation = LCB_SET;
-                scmd.exptime = exptime;
-                LCB_CMD_SET_KEY(&scmd, opinfo.m_key.c_str(), opinfo.m_key.size());
-                LCB_CMD_SET_VALUEIOV(&scmd, &opinfo.m_valuefrags[0], opinfo.m_valuefrags.size());
-                error = lcb_store3(instance, this, &scmd);
-                break;
-            }
-            case NextOp::GET: {
-                lcb_CMDGET gcmd = { 0 };
-                LCB_CMD_SET_KEY(&gcmd, opinfo.m_key.c_str(), opinfo.m_key.size());
-                gcmd.exptime = exptime;
-                error = lcb_get3(instance, this, &gcmd);
-                break;
-            }
-            case NextOp::SDSTORE:
-            case NextOp::SDGET: {
-                lcb_CMDSUBDOC sdcmd = { 0 };
-                if (opinfo.m_mode == NextOp::SDSTORE) {
-                    sdcmd.exptime = exptime;
-                }
-                LCB_CMD_SET_KEY(&sdcmd, opinfo.m_key.c_str(), opinfo.m_key.size());
-                sdcmd.specs = &opinfo.m_specs[0];
-                sdcmd.nspecs = opinfo.m_specs.size();
-                error = lcb_subdoc3(instance, this, &sdcmd);
-                break;
-            }
-            case NextOp::NOOP: {
-                lcb_CMDNOOP ncmd = { 0 };
-                error = lcb_noop3(instance, this, &ncmd);
-                break;
-            }
-            }
-
-            if (error != LCB_SUCCESS) {
-                hasItems = false;
-                log("Failed to schedule operation: [0x%x] %s", error, lcb_strerror(instance, error));
-            } else {
-                hasItems = true;
-            }
+            hasItems = scheduleNextOperation();
         }
         if (hasItems) {
+            error = LCB_SUCCESS;
             lcb_sched_leave(instance);
             lcb_wait(instance);
-            if (error != LCB_SUCCESS) {
-                log("Operation(s) failed: [0x%x] %s", error, lcb_strerror(instance, error));
-            }
         } else {
             lcb_sched_fail(instance);
         }
+        purgeRetryQueue();
+    }
+
+    void purgeRetryQueue() {
+        NextOp opinfo;
+        InstanceCookie *cookie = InstanceCookie::get(instance);
 
         while (!retryq.empty()) {
+            unsigned exptime = config.getExptime();
             lcb_sched_enter(instance);
             while (!retryq.empty()) {
                 opinfo = retryq.front();
                 retryq.pop();
-                lcb_CMDSTORE scmd = { 0 };
+                lcb_CMDSTOREDUR scmd = { 0 };
                 scmd.operation = LCB_SET;
                 scmd.exptime = exptime;
                 LCB_CMD_SET_KEY(&scmd, opinfo.m_key.c_str(), opinfo.m_key.size());
                 LCB_CMD_SET_VALUEIOV(&scmd, &opinfo.m_valuefrags[0], opinfo.m_valuefrags.size());
-                error = lcb_store3(instance, this, &scmd);
+                if (config.persistTo > 0 || config.replicateTo > 0) {
+                    scmd.persist_to = config.persistTo;
+                    scmd.replicate_to = config.replicateTo;
+                    error = lcb_storedur3(instance, NULL, &scmd);
+                } else {
+                    error = lcb_store3(instance, NULL, reinterpret_cast<lcb_CMDSTORE*>(&scmd));
+                }
+                cookie->stats.retried++;
             }
             lcb_sched_leave(instance);
             lcb_wait(instance);
@@ -673,11 +745,77 @@ public:
         }
     }
 
+    bool scheduleNextOperation()
+    {
+        NextOp opinfo;
+        unsigned exptime = config.getExptime();
+        gen->setNextOp(opinfo);
+
+        switch (opinfo.m_mode) {
+        case NextOp::STORE: {
+            if (!gen->inPopulation() && config.lockTime > 0) {
+                lcb_CMDGET gcmd = { 0 };
+                LCB_CMD_SET_KEY(&gcmd, opinfo.m_key.c_str(), opinfo.m_key.size());
+                gcmd.lock = config.lockTime;
+                error = lcb_get3(instance, (void *)OPFLAGS_LOCKED, &gcmd);
+            } else {
+                lcb_CMDSTOREDUR scmd = { 0 };
+                scmd.operation = LCB_SET;
+                scmd.exptime = exptime;
+                if (config.writeJson()) {
+                    scmd.datatype = LCB_VALUE_F_JSON;
+                }
+                LCB_CMD_SET_KEY(&scmd, opinfo.m_key.c_str(), opinfo.m_key.size());
+                LCB_CMD_SET_VALUEIOV(&scmd, &opinfo.m_valuefrags[0], opinfo.m_valuefrags.size());
+                if (config.persistTo > 0 || config.replicateTo > 0) {
+                    scmd.persist_to = config.persistTo;
+                    scmd.replicate_to = config.replicateTo;
+                    error = lcb_storedur3(instance, NULL, &scmd);
+                } else {
+                    error = lcb_store3(instance, NULL, reinterpret_cast<lcb_CMDSTORE*>(&scmd));
+                }
+            }
+            break;
+        }
+        case NextOp::GET: {
+            lcb_CMDGET gcmd = { 0 };
+            LCB_CMD_SET_KEY(&gcmd, opinfo.m_key.c_str(), opinfo.m_key.size());
+            gcmd.exptime = exptime;
+            error = lcb_get3(instance, this, &gcmd);
+            break;
+        }
+        case NextOp::SDSTORE:
+        case NextOp::SDGET: {
+            lcb_CMDSUBDOC sdcmd = { 0 };
+            if (opinfo.m_mode == NextOp::SDSTORE) {
+                sdcmd.exptime = exptime;
+            }
+            LCB_CMD_SET_KEY(&sdcmd, opinfo.m_key.c_str(), opinfo.m_key.size());
+            sdcmd.specs = &opinfo.m_specs[0];
+            sdcmd.nspecs = opinfo.m_specs.size();
+            error = lcb_subdoc3(instance, NULL, &sdcmd);
+            break;
+        }
+        case NextOp::NOOP: {
+            lcb_CMDNOOP ncmd = { 0 };
+            error = lcb_noop3(instance, NULL, &ncmd);
+            break;
+        }
+        }
+
+        if (error != LCB_SUCCESS) {
+            log("Failed to schedule operation: [0x%x] %s", error, lcb_strerror(instance, error));
+            return false;
+        } else {
+            return true;
+        }
+    }
+
     bool run() {
         do {
             singleLoop();
 
-            if (config.isTimings()) {
+            if (config.numTimings() > 1) {
                 InstanceCookie::dumpTimings(instance, gen->getStageString());
             }
             if (config.params.shouldDump()) {
@@ -689,7 +827,7 @@ public:
 
         } while (!config.isLoopDone(++niter));
 
-        if (config.isTimings()) {
+        if (config.numTimings() > 1) {
             InstanceCookie::dumpTimings(instance, gen->getStageString(), true);
         }
         return true;
@@ -702,13 +840,25 @@ public:
         retryq.push(op);
     }
 
+    void populateIov(uint32_t seq, vector<lcb_IOV>& iov_out)
+    {
+        gen->populateIov(seq, iov_out);
+    }
+
+
 #ifndef WIN32
     pthread_t thr;
 #endif
 
+    lcb_t getInstance() {
+        return instance;
+    }
+
 protected:
     // the callback methods needs to be able to set the error handler..
     friend void operationCallback(lcb_t, int, const lcb_RESPBASE*);
+    friend void storeCallback(lcb_t, int, const lcb_RESPBASE *);
+
     Histogram histogram;
 
     void setError(lcb_error_t e) { error = e; }
@@ -743,52 +893,195 @@ private:
     std::queue<NextOp> retryq;
 };
 
-static void operationCallback(lcb_t, int cbtype, const lcb_RESPBASE *resp)
+static void updateOpsPerSecDisplay()
 {
-    ThreadContext *tc;
 
-    tc = const_cast<ThreadContext *>(reinterpret_cast<const ThreadContext *>(resp->cookie));
-    if (cbtype == LCB_CALLBACK_STORE && resp->rc != LCB_SUCCESS && tc->inPopulation()) {
+    static time_t start_time = time(NULL);
+    static int is_tty =
+#ifdef WIN32
+        0;
+#else
+        isatty(STDERR_FILENO);
+#endif
+    static volatile unsigned long nops = 0;
+    time_t now = time(NULL);
+    time_t nsecs = now - start_time;
+    if (!nsecs) { nsecs = 1; }
+    unsigned long ops_sec = nops / nsecs;
+    if (++nops % 10000 == 0) {
+	    fprintf(stderr, "OPS/SEC: %10lu%c", ops_sec, is_tty ? '\r' : '\n');
+    }
+}
+
+static void updateStats(InstanceCookie *cookie, lcb_error_t rc)
+{
+    cookie->stats.total++;
+    switch (rc) {
+    case LCB_ETMPFAIL:
+        cookie->stats.etmpfail++;
+        break;
+    case LCB_KEY_EEXISTS:
+        cookie->stats.eexist++;
+        break;
+    case LCB_ETIMEDOUT:
+        cookie->stats.etimeout++;
+        break;
+    default:
+        break;
+    }
+}
+
+static void operationCallback(lcb_t instance, int cbtype, const lcb_RESPBASE *resp)
+{
+    InstanceCookie *cookie = InstanceCookie::get(instance);
+    ThreadContext *tc = cookie->getContext();
+    tc->setError(resp->rc);
+    updateStats(cookie, resp->rc);
+
+    uintptr_t flags = 0;
+    if (resp->cookie) {
+        flags = (uintptr_t)resp->cookie;
+    }
+    bool done = true;
+    string key((const char*)resp->key, resp->nkey);
+    uint32_t seqno = atoi(key.c_str());
+    if (cbtype == LCB_CALLBACK_GET && (flags & OPFLAGS_LOCKED)) {
+        if (resp->rc == LCB_SUCCESS) {
+            lcb_CMDSTOREDUR scmd = { 0 };
+            vector<lcb_IOV> valuefrags;
+            scmd.operation = LCB_SET;
+            scmd.exptime = config.getExptime();
+            scmd.cas = resp->cas;
+            tc->populateIov(seqno, valuefrags);
+            LCB_CMD_SET_KEY(&scmd, resp->key, resp->nkey);
+            LCB_CMD_SET_VALUEIOV(&scmd, &valuefrags[0], valuefrags.size());
+            if (config.persistTo > 0 || config.replicateTo > 0) {
+                scmd.persist_to = config.persistTo;
+                scmd.replicate_to = config.replicateTo;
+                lcb_storedur3(instance, NULL, &scmd);
+            } else {
+                lcb_store3(instance, NULL, reinterpret_cast<lcb_CMDSTORE*>(&scmd));
+            }
+            done = false;
+        } else if (resp->rc == LCB_ETMPFAIL) {
+            NextOp op;
+            op.m_mode = NextOp::STORE;
+            op.m_key = key;
+            op.m_seqno = seqno;
+            tc->retry(op);
+            done = false;
+        }
+    }
+
+    if (done) {
+        tc->checkin(seqno);
+    }
+    updateOpsPerSecDisplay();
+}
+
+static void storeCallback(lcb_t instance, int, const lcb_RESPBASE *resp)
+{
+    InstanceCookie *cookie = InstanceCookie::get(instance);
+    ThreadContext *tc = cookie->getContext();
+    tc->setError(resp->rc);
+    updateStats(cookie, resp->rc);
+
+    string key((const char*)resp->key, resp->nkey);
+    uint32_t seqno = atoi(key.c_str());
+    if (resp->rc != LCB_SUCCESS && tc->inPopulation()) {
         NextOp op;
         op.m_mode = NextOp::STORE;
-        op.m_key.assign((char *)resp->key, resp->nkey);
-        op.m_seqno = atoi(op.m_key.c_str());
+        op.m_key = key;
+        op.m_seqno = seqno;
         tc->retry(op);
     } else {
-        tc->setError(resp->rc);
+        tc->checkin(seqno);
     }
 
-#ifndef WIN32
-    static volatile unsigned long nops = 1;
-    static time_t start_time = time(NULL);
-    static int is_tty = isatty(STDERR_FILENO);
-    if (is_tty) {
-        if (++nops % 1000 == 0) {
-            time_t now = time(NULL);
-            time_t nsecs = now - start_time;
-            if (!nsecs) { nsecs = 1; }
-            unsigned long ops_sec = nops / nsecs;
-            fprintf(stderr, "OPS/SEC: %10lu\r", ops_sec);
-        }
-    }
-#endif
+    updateOpsPerSecDisplay();
 }
 
-
 std::list<ThreadContext *> contexts;
 
 extern "C" {
-    typedef void (*handler_t)(int);
+typedef void (*handler_t)(int);
+
+static void dump_metrics(void)
+{
+    std::list<ThreadContext *>::iterator it;
+    for (it = contexts.begin(); it != contexts.end(); ++it) {
+        lcb_t instance = (*it)->getInstance();
+        lcb_CMDDIAG req = {};
+        req.options = LCB_PINGOPT_F_JSONPRETTY;
+        lcb_diag(instance, NULL, &req);
+        if (config.numTimings() > 0) {
+            InstanceCookie::dumpTimings(instance);
+        }
+    }
 }
 
 #ifndef WIN32
+static void diag_callback(lcb_t instance, int, const lcb_RESPBASE *rb)
+{
+    const lcb_RESPDIAG *resp = (const lcb_RESPDIAG *)rb;
+    if (resp->rc != LCB_SUCCESS) {
+        fprintf(stderr, "%p, diag failed: %s\n", (void *)instance, lcb_strerror(NULL, resp->rc));
+    } else {
+        if (resp->njson) {
+            fprintf(stderr, "\n%.*s", (int)resp->njson, resp->json);
+        }
+
+        {
+            InstanceCookie *cookie = InstanceCookie::get(instance);
+            lcb_METRICS* metrics;
+            size_t ii;
+            lcb_cntl(instance, LCB_CNTL_GET, LCB_CNTL_METRICS, &metrics);
+
+            fprintf(stderr, "%p: total: %lu, etmpfail: %lu, eexist: %lu, etimeout: %lu, retried: %lu, rq: %lu\n",
+                    (void *)instance,
+                    (unsigned long)cookie->stats.total,
+                    (unsigned long)cookie->stats.etmpfail,
+                    (unsigned long)cookie->stats.eexist,
+                    (unsigned long)cookie->stats.etimeout,
+                    (unsigned long)cookie->stats.retried,
+                    (unsigned long)metrics->packets_retried);
+            for (ii = 0; ii < metrics->nservers; ii++) {
+                fprintf(stderr, "  [srv-%d] snt: %lu, rcv: %lu, q: %lu, err: %lu, tmo: %lu, nmv: %lu, orph: %lu\n",
+                        (int)ii,
+                        (unsigned long)metrics->servers[ii]->packets_sent,
+                        (unsigned long)metrics->servers[ii]->packets_read,
+                        (unsigned long)metrics->servers[ii]->packets_queued,
+                        (unsigned long)metrics->servers[ii]->packets_errored,
+                        (unsigned long)metrics->servers[ii]->packets_timeout,
+                        (unsigned long)metrics->servers[ii]->packets_nmv,
+                        (unsigned long)metrics->servers[ii]->packets_ownerless);
+            }
+        }
+    }
+}
+
+static void sigquit_handler(int)
+{
+    dump_metrics();
+    signal(SIGQUIT, sigquit_handler); // Reinstall
+}
+
+static void setup_sigquit_handler()
+{
+    struct sigaction action;
+    sigemptyset(&action.sa_mask);
+    action.sa_handler = sigquit_handler;
+    action.sa_flags = 0;
+    sigaction(SIGQUIT, &action, NULL);
+}
+
 static void sigint_handler(int)
 {
     static int ncalled = 0;
     ncalled++;
 
     if (ncalled < 2) {
-        log("Termination requested. Waiting threads to finish. Ctrl-C to force termination.");
+        log("\nTermination requested. Waiting threads to finish. Ctrl-C to force termination.");
         signal(SIGINT, sigint_handler); // Reinstall
         config.maxCycles = 0;
         return;
@@ -811,9 +1104,7 @@ static void setup_sigint_handler()
     sigaction(SIGINT, &action, NULL);
 }
 
-extern "C" {
 static void* thread_worker(void*);
-}
 
 static void start_worker(ThreadContext *ctx)
 {
@@ -837,12 +1128,12 @@ static void join_worker(ThreadContext *ctx)
 }
 
 #else
+static void setup_sigquit_handler() {}
 static void setup_sigint_handler() {}
 static void start_worker(ThreadContext *ctx) { ctx->run(); }
 static void join_worker(ThreadContext *ctx) { (void)ctx; }
 #endif
 
-extern "C" {
 static void *thread_worker(void *arg)
 {
     ThreadContext *ctx = static_cast<ThreadContext *>(arg);
@@ -855,6 +1146,7 @@ int main(int argc, char **argv)
 {
     int exit_code = EXIT_SUCCESS;
     setup_sigint_handler();
+    setup_sigquit_handler();
 
     Parser parser("cbc-pillowfight");
     try {
@@ -890,14 +1182,26 @@ int main(int argc, char **argv)
             log("Failed to create instance: %s", lcb_strerror(NULL, error));
             exit(EXIT_FAILURE);
         }
-        lcb_install_callback3(instance, LCB_CALLBACK_STORE, operationCallback);
+        lcb_install_callback3(instance, LCB_CALLBACK_STOREDUR, storeCallback);
+        lcb_install_callback3(instance, LCB_CALLBACK_STORE, storeCallback);
         lcb_install_callback3(instance, LCB_CALLBACK_GET, operationCallback);
         lcb_install_callback3(instance, LCB_CALLBACK_SDMUTATE, operationCallback);
         lcb_install_callback3(instance, LCB_CALLBACK_SDLOOKUP, operationCallback);
         lcb_install_callback3(instance, LCB_CALLBACK_NOOP, operationCallback);
+#ifndef WIN32
+        lcb_install_callback3(instance, LCB_CALLBACK_DIAG, diag_callback);
+        {
+            int activate = 1;
+            lcb_cntl(instance, LCB_CNTL_SET, LCB_CNTL_METRICS, &activate);
+        }
+#endif
         cp.doCtls(instance);
+        if (config.useCollections()) {
+            int use = 1;
+            lcb_cntl(instance, LCB_CNTL_SET, LCB_CNTL_USE_COLLECTIONS, &use);
+        }
 
-        new InstanceCookie(instance);
+        InstanceCookie *cookie = new InstanceCookie(instance);
 
         lcb_connect(instance);
         lcb_wait(instance);
@@ -910,6 +1214,7 @@ int main(int argc, char **argv)
         }
 
         ThreadContext *ctx = new ThreadContext(instance, ii);
+        cookie->setContext(ctx);
         contexts.push_back(ctx);
         start_worker(ctx);
     }
@@ -918,5 +1223,8 @@ int main(int argc, char **argv)
             it != contexts.end(); ++it) {
         join_worker(*it);
     }
+    if (config.numTimings() > 0) {
+        dump_metrics();
+    }
     return exit_code;
 }