libcouchbase 0.0.7 → 0.0.8

data/ext/libcouchbase/src/mcserver/mcserver.c

@@ -1,784 +0,0 @@
-/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
-/*
- *     Copyright 2011-2014 Couchbase, Inc.
- *
- *   Licensed under the Apache License, Version 2.0 (the "License");
- *   you may not use this file except in compliance with the License.
- *   You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *   Unless required by applicable law or agreed to in writing, software
- *   distributed under the License is distributed on an "AS IS" BASIS,
- *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *   See the License for the specific language governing permissions and
- *   limitations under the License.
- */
-
-#include "internal.h"
-#include "logging.h"
-#include "vbucket/aliases.h"
-#include "settings.h"
-#include "negotiate.h"
-#include "bucketconfig/clconfig.h"
-#include "mc/mcreq-flush-inl.h"
-#include <lcbio/ssl.h>
-#include "ctx-log-inl.h"
-
-#define LOGARGS(c, lvl) (c)->settings, "server", LCB_LOG_##lvl, __FILE__, __LINE__
-#define LOGFMT "<%s:%s> (SRV=%p,IX=%d) "
-#define LOGID(server) get_ctx_host(server->connctx), get_ctx_port(server->connctx), (void*)server, server->pipeline.index
-#define MCREQ_MAXIOV 32
-#define LCBCONN_UNWANT(conn, flags) (conn)->want &= ~(flags)
-
-typedef enum {
-    /* There are no known errored commands on this server */
-    S_CLEAN,
-
-    /* In the process of draining remaining commands to be flushed. The commands
-     * being drained may have already been rescheduled to another server or
-     * placed inside the error queue, but are pending being flushed. This will
-     * only happen in completion-style I/O plugins. When this state is in effect,
-     * subsequent attempts to connect will be blocked until all commands have
-     * been properly drained.
-     */
-    S_ERRDRAIN,
-
-    /* The server object has been closed, either because it has been removed
-     * from the cluster or because the related lcb_t has been destroyed.
-     */
-    S_CLOSED
-} mcserver_STATE;
-
-static int check_closed(mc_SERVER *);
-static void start_errored_ctx(mc_SERVER *server, mcserver_STATE next_state);
-static void finalize_errored_ctx(mc_SERVER *server);
-static void on_error(lcbio_CTX *ctx, lcb_error_t err);
-static void server_socket_failed(mc_SERVER *server, lcb_error_t err);
-
-static void
-on_flush_ready(lcbio_CTX *ctx)
-{
-    mc_SERVER *server = lcbio_ctx_data(ctx);
-    nb_IOV iov[MCREQ_MAXIOV];
-    int ready;
-
-    do {
-        int niov = 0;
-        unsigned nb;
-        nb = mcreq_flush_iov_fill(&server->pipeline, iov, MCREQ_MAXIOV, &niov);
-        if (!nb) {
-            return;
-        }
-        ready = lcbio_ctx_put_ex(ctx, (lcb_IOV *)iov, niov, nb);
-    } while (ready);
-    lcbio_ctx_wwant(ctx);
-}
-
-static void
-on_flush_done(lcbio_CTX *ctx, unsigned expected, unsigned actual)
-{
-    mc_SERVER *server = lcbio_ctx_data(ctx);
-    lcb_U64 now = 0;
-    if (server->settings->readj_ts_wait) {
-        now = gethrtime();
-    }
-
-    mcreq_flush_done_ex(&server->pipeline, actual, expected, now);
-    check_closed(server);
-}
-
-void
-mcserver_flush(mc_SERVER *server)
-{
-    /** Call into the wwant stuff.. */
-    if (!server->connctx->rdwant) {
-        lcbio_ctx_rwant(server->connctx, 24);
-    }
-
-    lcbio_ctx_wwant(server->connctx);
-    lcbio_ctx_schedule(server->connctx);
-
-    if (!lcbio_timer_armed(server->io_timer)) {
-        /**
-         * XXX: Maybe use get_next_timeout(), although here we can assume
-         * that a command was just scheduled
-         */
-        lcbio_timer_rearm(server->io_timer, MCSERVER_TIMEOUT(server));
-    }
-}
-
-LIBCOUCHBASE_API
-void
-lcb_sched_flush(lcb_t instance)
-{
-    unsigned ii;
-    for (ii = 0; ii < LCBT_NSERVERS(instance); ii++) {
-        mc_SERVER *server = LCBT_GET_SERVER(instance, ii);
-
-        if (!mcserver_has_pending(server)) {
-            continue;
-        }
-        server->pipeline.flush_start(&server->pipeline);
-    }
-}
-
-/**
- * Invoked when get a NOT_MY_VBUCKET response. If the response contains a JSON
- * payload then we refresh the configuration with it.
- *
- * This function returns 1 if the operation was successfully rescheduled;
- * otherwise it returns 0. If it returns 0 then we give the error back to the
- * user.
- */
-static int
-handle_nmv(mc_SERVER *oldsrv, packet_info *resinfo, mc_PACKET *oldpkt)
-{
-    mc_PACKET *newpkt;
-    protocol_binary_request_header hdr;
-    lcb_error_t err = LCB_ERROR;
-    lcb_t instance = oldsrv->instance;
-    lcb_U16 vbid;
-    clconfig_provider *cccp = lcb_confmon_get_provider(instance->confmon,
-        LCB_CLCONFIG_CCCP);
-
-    mcreq_read_hdr(oldpkt, &hdr);
-    vbid = ntohs(hdr.request.vbucket);
-    lcb_log(LOGARGS(oldsrv, WARN), LOGFMT "NOT_MY_VBUCKET. Packet=%p (S=%u). VBID=%u", LOGID(oldsrv), (void*)oldpkt, oldpkt->opaque, vbid);
-
-    /* Notify of new map */
-    lcb_vbguess_remap(instance, vbid, oldsrv->pipeline.index);
-
-    if (PACKET_NBODY(resinfo) && cccp->enabled) {
-        lcb_string s;
-
-        lcb_string_init(&s);
-        lcb_string_append(&s, PACKET_VALUE(resinfo), PACKET_NVALUE(resinfo));
-        err = lcb_cccp_update(cccp, mcserver_get_host(oldsrv), &s);
-        lcb_string_release(&s);
-    }
-
-    if (err != LCB_SUCCESS) {
-        int bs_options;
-        if (instance->cur_configinfo->origin == LCB_CLCONFIG_CCCP) {
-            /**
-             * XXX: Not enough to see if cccp was enabled, since cccp might
-             * be requested by a user, but would still not actually be active
-             * for clusters < 2.5 If our current config is from CCCP
-             * then we can be fairly certain that CCCP is indeed working.
-             *
-             * For this reason, we don't use if (cccp->enabled) {...}
-             */
-            bs_options = LCB_BS_REFRESH_THROTTLE;
-        } else {
-            bs_options = LCB_BS_REFRESH_ALWAYS;
-        }
-        lcb_bootstrap_common(instance, bs_options);
-    }
-
-    if (!lcb_should_retry(oldsrv->settings, oldpkt, LCB_NOT_MY_VBUCKET)) {
-        return 0;
-    }
-
-    /** Reschedule the packet again .. */
-    newpkt = mcreq_renew_packet(oldpkt);
-    newpkt->flags &= ~MCREQ_STATE_FLAGS;
-    lcb_retryq_nmvadd(instance->retryq, (mc_EXPACKET*)newpkt);
-    return 1;
-}
-
-#define PKT_READ_COMPLETE 1
-#define PKT_READ_PARTIAL 0
-
-/* This function is called within a loop to process a single packet.
- *
- * If a full packet is available, it will process the packet and return
- * PKT_READ_COMPLETE, resulting in the `on_read()` function calling this
- * function in a loop.
- *
- * When a complete packet is not available, PKT_READ_PARTIAL will be returned
- * and the `on_read()` loop will exit, scheduling any required pending I/O.
- */
-static int
-try_read(lcbio_CTX *ctx, mc_SERVER *server, rdb_IOROPE *ior)
-{
-    packet_info info_s, *info = &info_s;
-    mc_PACKET *request;
-    mc_PIPELINE *pl = &server->pipeline;
-    unsigned pktsize = 24, is_last = 1;
-
-    #define RETURN_NEED_MORE(n) \
-        if (mcserver_has_pending(server)) { \
-            lcbio_ctx_rwant(ctx, n); \
-        } \
-        return PKT_READ_PARTIAL; \
-
-    #define DO_ASSIGN_PAYLOAD() \
-        rdb_consumed(ior, sizeof(info->res.bytes)); \
-        if (PACKET_NBODY(info)) { \
-            info->payload = rdb_get_consolidated(ior, PACKET_NBODY(info)); \
-        } {
-
-    #define DO_SWALLOW_PAYLOAD() \
-        } if (PACKET_NBODY(info)) { \
-            rdb_consumed(ior, PACKET_NBODY(info)); \
-        }
-
-    if (rdb_get_nused(ior) < pktsize) {
-        RETURN_NEED_MORE(pktsize)
-    }
-
-    /* copy bytes into the info structure */
-    rdb_copyread(ior, info->res.bytes, sizeof info->res.bytes);
-
-    pktsize += PACKET_NBODY(info);
-    if (rdb_get_nused(ior) < pktsize) {
-        RETURN_NEED_MORE(pktsize);
-    }
-
-    /* Find the packet */
-    if (PACKET_OPCODE(info) == PROTOCOL_BINARY_CMD_STAT && PACKET_NKEY(info) != 0) {
-        is_last = 0;
-        request = mcreq_pipeline_find(pl, PACKET_OPAQUE(info));
-    } else {
-        is_last = 1;
-        request = mcreq_pipeline_remove(pl, PACKET_OPAQUE(info));
-    }
-
-    if (!request) {
-        lcb_log(LOGARGS(server, WARN), LOGFMT "Found stale packet (OP=0x%x, RC=0x%x, SEQ=%u)", LOGID(server), PACKET_OPCODE(info), PACKET_STATUS(info), PACKET_OPAQUE(info));
-        rdb_consumed(ior, pktsize);
-        return PKT_READ_COMPLETE;
-    }
-
-    if (PACKET_STATUS(info) == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET) {
-        /* consume the header */
-        DO_ASSIGN_PAYLOAD()
-        if (!handle_nmv(server, info, request)) {
-            mcreq_dispatch_response(pl, request, info, LCB_NOT_MY_VBUCKET);
-        }
-        DO_SWALLOW_PAYLOAD()
-        goto GT_DONE;
-    }
-
-    /* Figure out if the request is 'ufwd' or not */
-    if (!(request->flags & MCREQ_F_UFWD)) {
-        DO_ASSIGN_PAYLOAD();
-        info->bufh = rdb_get_first_segment(ior);
-        mcreq_dispatch_response(pl, request, info, LCB_SUCCESS);
-        DO_SWALLOW_PAYLOAD()
-
-    } else {
-        /* figure out how many buffers we want to use as an upper limit for the
-         * IOV arrays. Currently we'll keep it simple and ensure the entire
-         * response is contiguous. */
-        lcb_PKTFWDRESP resp = { 0 };
-        rdb_ROPESEG *segs;
-        nb_IOV iov;
-
-        rdb_consolidate(ior, pktsize);
-        rdb_refread_ex(ior, &iov, &segs, 1, pktsize);
-
-        resp.bufs = &segs;
-        resp.iovs = (lcb_IOV*)&iov;
-        resp.nitems = 1;
-        resp.header = info->res.bytes;
-        server->instance->callbacks.pktfwd(
-            server->instance, MCREQ_PKT_COOKIE(request), LCB_SUCCESS, &resp);
-        rdb_consumed(ior, pktsize);
-    }
-
-    GT_DONE:
-    if (is_last) {
-        mcreq_packet_handled(pl, request);
-    }
-    return PKT_READ_COMPLETE;
-}
-
-static void
-on_read(lcbio_CTX *ctx, unsigned nb)
-{
-    int rv;
-    mc_SERVER *server = lcbio_ctx_data(ctx);
-    rdb_IOROPE *ior = &ctx->ior;
-
-    if (check_closed(server)) {
-        return;
-    }
-
-    while ((rv = try_read(ctx, server, ior)) == PKT_READ_COMPLETE);
-    lcbio_ctx_schedule(ctx);
-    lcb_maybe_breakout(server->instance);
-
-    (void)nb;
-}
-
-LCB_INTERNAL_API
-int
-mcserver_has_pending(mc_SERVER *server)
-{
-    return !SLLIST_IS_EMPTY(&server->pipeline.requests);
-}
-
-static void flush_noop(mc_PIPELINE *pipeline) {
-    (void)pipeline;
-}
-static void server_connect(mc_SERVER *server);
-
-typedef enum {
-    REFRESH_ALWAYS,
-    REFRESH_ONFAILED,
-    REFRESH_NEVER
-} mc_REFRESHPOLICY;
-
-static int
-maybe_retry(mc_PIPELINE *pipeline, mc_PACKET *pkt, lcb_error_t err)
-{
-    mc_SERVER *srv = (mc_SERVER *)pipeline;
-    mc_PACKET *newpkt;
-    lcb_t instance = pipeline->parent->cqdata;
-    lcbvb_DISTMODE dist_t = lcbvb_get_distmode(pipeline->parent->config);
-
-    if (dist_t != LCBVB_DIST_VBUCKET) {
-        /** memcached bucket */
-        return 0;
-    }
-    if (!lcb_should_retry(srv->settings, pkt, err)) {
-        return 0;
-    }
-
-    newpkt = mcreq_renew_packet(pkt);
-    newpkt->flags &= ~MCREQ_STATE_FLAGS;
-    lcb_retryq_add(instance->retryq, (mc_EXPACKET *)newpkt, err);
-    return 1;
-}
-
-static void
-fail_callback(mc_PIPELINE *pipeline, mc_PACKET *pkt, lcb_error_t err, void *arg)
-{
-    int rv;
-    mc_SERVER *server = (mc_SERVER *)pipeline;
-    packet_info info;
-    protocol_binary_request_header hdr;
-    protocol_binary_response_header *res = &info.res;
-
-    if (maybe_retry(pipeline, pkt, err)) {
-        return;
-    }
-
-    if (err == LCB_AUTH_ERROR) {
-        /* In-situ auth errors are actually dead servers. Let's provide this
-         * as the actual error code. */
-        err = LCB_MAP_CHANGED;
-    }
-
-    if (err == LCB_ETIMEDOUT) {
-        lcb_error_t tmperr = lcb_retryq_origerr(pkt);
-        if (tmperr != LCB_SUCCESS) {
-            err = tmperr;
-        }
-    }
-
-    memset(&info, 0, sizeof(info));
-    memcpy(hdr.bytes, SPAN_BUFFER(&pkt->kh_span), sizeof(hdr.bytes));
-
-    res->response.status = ntohs(PROTOCOL_BINARY_RESPONSE_EINVAL);
-    res->response.opcode = hdr.request.opcode;
-    res->response.opaque = hdr.request.opaque;
-
-    lcb_log(LOGARGS(server, WARN), LOGFMT "Failing command (pkt=%p, opaque=%lu, opcode=0x%x) with error 0x%x", LOGID(server), (void*)pkt, (unsigned long)pkt->opaque, hdr.request.opcode, err);
-    rv = mcreq_dispatch_response(pipeline, pkt, &info, err);
-    lcb_assert(rv == 0);
-    (void)arg;
-}
-
-static int
-purge_single_server(mc_SERVER *server, lcb_error_t error,
-                    hrtime_t thresh, hrtime_t *next, int policy)
-{
-    unsigned affected;
-    mc_PIPELINE *pl = &server->pipeline;
-
-    if (thresh) {
-        affected = mcreq_pipeline_timeout(
-                pl, error, fail_callback, NULL, thresh, next);
-
-    } else {
-        mcreq_pipeline_fail(pl, error, fail_callback, NULL);
-        affected = -1;
-    }
-
-    if (policy == REFRESH_NEVER) {
-        return affected;
-    }
-
-    if (affected || policy == REFRESH_ALWAYS) {
-        lcb_bootstrap_common(server->instance,
-            LCB_BS_REFRESH_THROTTLE|LCB_BS_REFRESH_INCRERR);
-    }
-    return affected;
-}
-
-static void flush_errdrain(mc_PIPELINE *pipeline)
-{
-    /* Called when we are draining errors. */
-    mc_SERVER *server = (mc_SERVER *)pipeline;
-    if (!lcbio_timer_armed(server->io_timer)) {
-        lcbio_timer_rearm(server->io_timer, MCSERVER_TIMEOUT(server));
-    }
-}
-
-void
-mcserver_fail_chain(mc_SERVER *server, lcb_error_t err)
-{
-    purge_single_server(server, err, 0, NULL, REFRESH_NEVER);
-}
-
-
-static uint32_t
-get_next_timeout(mc_SERVER *server)
-{
-    hrtime_t now, expiry, diff;
-    mc_PACKET *pkt = mcreq_first_packet(&server->pipeline);
-
-    if (!pkt) {
-        return MCSERVER_TIMEOUT(server);
-    }
-
-    now = gethrtime();
-    expiry = MCREQ_PKT_RDATA(pkt)->start + LCB_US2NS(MCSERVER_TIMEOUT(server));
-    if (expiry <= now) {
-        diff = 0;
-    } else {
-        diff = expiry - now;
-    }
-
-    return LCB_NS2US(diff);
-}
-
-static void
-timeout_server(void *arg)
-{
-    mc_SERVER *server = arg;
-    hrtime_t now, min_valid, next_ns = 0;
-    uint32_t next_us;
-    int npurged;
-
-    now = gethrtime();
-    min_valid = now - LCB_US2NS(MCSERVER_TIMEOUT(server));
-    npurged = purge_single_server(server,
-        LCB_ETIMEDOUT, min_valid, &next_ns, REFRESH_ONFAILED);
-    if (npurged) {
-        lcb_log(LOGARGS(server, ERROR), LOGFMT "Server timed out. Some commands have failed", LOGID(server));
-    }
-
-    next_us = get_next_timeout(server);
-    lcb_log(LOGARGS(server, DEBUG), LOGFMT "Scheduling next timeout for %u ms", LOGID(server), next_us / 1000);
-    lcbio_timer_rearm(server->io_timer, next_us);
-    lcb_maybe_breakout(server->instance);
-}
-
-static int
-maybe_retry_timeout_connection(mc_SERVER *server, lcb_error_t err)
-{
-    lcb_U32 next_tmo;
-    if (err != LCB_ETIMEDOUT) {
-        return 0; /* not a timeout */
-    }
-    if (!server->settings->readj_ts_wait) {
-        return 0; /* normal timeout behavior */
-    }
-    if (!mcserver_has_pending(server)) {
-        return 0; /* nothing pending */
-    }
-
-    next_tmo = get_next_timeout(server);
-    if (next_tmo < MCSERVER_TIMEOUT(server) / 2) {
-        /* Ideally we'd have a fuzz interval to shave off the actual timeout,
-         * since there will inevitably be some time taken off the next timeout */
-        return 0;
-    }
-
-    lcb_log(LOGARGS(server, INFO), LOGFMT "Retrying connection. Assuming timeout because of stalled event loop", LOGID(server));
-    server_connect(server);
-    return 1;
-}
-
-static void
-on_connected(lcbio_SOCKET *sock, void *data, lcb_error_t err, lcbio_OSERR syserr)
-{
-    mc_SERVER *server = data;
-    lcbio_CTXPROCS procs;
-    uint32_t tmo;
-    mc_pSESSINFO sessinfo = NULL;
-    LCBIO_CONNREQ_CLEAR(&server->connreq);
-
-    if (err != LCB_SUCCESS) {
-        lcb_log(LOGARGS(server, ERR), LOGFMT "Got error for connection! (OS=%d)", LOGID(server), syserr);
-        if (maybe_retry_timeout_connection(server, err)) {
-            return;
-        }
-        server_socket_failed(server, err);
-        return;
-    }
-
-    lcb_assert(sock);
-
-    /** Do we need sasl? */
-    sessinfo = mc_sess_get(sock);
-    if (sessinfo == NULL) {
-        mc_pSESSREQ sreq;
-        lcb_log(LOGARGS(server, TRACE), "<%s:%s> (SRV=%p) Session not yet negotiated. Negotiating", server->curhost->host, server->curhost->port, (void*)server);
-        sreq = mc_sessreq_start(sock, server->settings, MCSERVER_TIMEOUT(server),
-            on_connected, data);
-        LCBIO_CONNREQ_MKGENERIC(&server->connreq, sreq, mc_sessreq_cancel);
-        return;
-    } else {
-        server->compsupport = mc_sess_chkfeature(sessinfo,
-            PROTOCOL_BINARY_FEATURE_DATATYPE);
-        server->mutation_tokens = mc_sess_chkfeature(sessinfo,
-            PROTOCOL_BINARY_FEATURE_MUTATION_SEQNO);
-    }
-
-    procs.cb_err = on_error;
-    procs.cb_read = on_read;
-    procs.cb_flush_done = on_flush_done;
-    procs.cb_flush_ready = on_flush_ready;
-    server->connctx = lcbio_ctx_new(sock, server, &procs);
-    server->connctx->subsys = "memcached";
-    server->pipeline.flush_start = (mcreq_flushstart_fn)mcserver_flush;
-
-    tmo = get_next_timeout(server);
-    lcb_log(LOGARGS(server, DEBUG), LOGFMT "Setting initial timeout=%ums", LOGID(server), tmo/1000);
-    lcbio_timer_rearm(server->io_timer, get_next_timeout(server));
-    mcserver_flush(server);
-}
-
-static void
-server_connect(mc_SERVER *server)
-{
-    lcbio_pMGRREQ mr;
-    mr = lcbio_mgr_get(server->instance->memd_sockpool, server->curhost,
-                       MCSERVER_TIMEOUT(server), on_connected, server);
-    LCBIO_CONNREQ_MKPOOLED(&server->connreq, mr);
-    server->pipeline.flush_start = flush_noop;
-    server->state = S_CLEAN;
-}
-
-static void
-buf_done_cb(mc_PIPELINE *pl, const void *cookie, void *kbuf, void *vbuf)
-{
-    mc_SERVER *server = (mc_SERVER*)pl;
-    server->instance->callbacks.pktflushed(server->instance, cookie);
-    (void)kbuf; (void)vbuf;
-}
-
-mc_SERVER *
-mcserver_alloc(lcb_t instance, int ix)
-{
-    mc_SERVER *ret;
-    lcbvb_SVCMODE mode;
-    const char *datahost;
-
-    ret = calloc(1, sizeof(*ret));
-    if (!ret) {
-        return ret;
-    }
-
-    ret->instance = instance;
-    ret->settings = instance->settings;
-    ret->curhost = calloc(1, sizeof(*ret->curhost));
-    mode = ret->settings->sslopts & LCB_SSL_ENABLED
-            ? LCBVB_SVCMODE_SSL : LCBVB_SVCMODE_PLAIN;
-
-    lcb_settings_ref(ret->settings);
-    mcreq_pipeline_init(&ret->pipeline);
-    ret->pipeline.flush_start = (mcreq_flushstart_fn)server_connect;
-    ret->pipeline.buf_done_callback = buf_done_cb;
-
-    datahost = lcbvb_get_hostport(
-        LCBT_VBCONFIG(instance), ix, LCBVB_SVCTYPE_DATA, mode);
-    if (datahost) {
-        lcb_host_parsez(ret->curhost, datahost, LCB_CONFIG_MCD_PORT);
-    }
-    ret->io_timer = lcbio_timer_new(instance->iotable, ret, timeout_server);
-    return ret;
-}
-
-static void
-server_free(mc_SERVER *server)
-{
-    mcreq_pipeline_cleanup(&server->pipeline);
-
-    if (server->io_timer) {
-        lcbio_timer_destroy(server->io_timer);
-    }
-
-    free(server->curhost);
-    lcb_settings_unref(server->settings);
-    free(server);
-}
-
-static void
-close_cb(lcbio_SOCKET *sock, int reusable, void *arg)
-{
-    lcbio_ref(sock);
-    lcbio_mgr_discard(sock);
-    (void)reusable;(void)arg;
-}
-
-
-/**Marks any unflushed data inside this server as being already flushed. This
- * should be done within error handling. If subsequent data is flushed on this
- * pipeline to the same connection, the results are undefined. */
-static void
-release_unflushed_packets(mc_SERVER *server)
-{
-    unsigned toflush;
-    nb_IOV iov;
-    mc_PIPELINE *pl = &server->pipeline;
-    while ((toflush = mcreq_flush_iov_fill(pl, &iov, 1, NULL))) {
-        mcreq_flush_done(pl, toflush, toflush);
-    }
-}
-
-static void
-on_error(lcbio_CTX *ctx, lcb_error_t err)
-{
-    mc_SERVER *server = lcbio_ctx_data(ctx);
-    lcb_log(LOGARGS(server, WARN), LOGFMT "Got socket error 0x%x", LOGID(server), err);
-    if (check_closed(server)) {
-        return;
-    }
-    server_socket_failed(server, err);
-}
-
-/**Handle a socket error. This function will close the current connection
- * and trigger a failout of any pending commands.
- * This function triggers a configuration refresh */
-static void
-server_socket_failed(mc_SERVER *server, lcb_error_t err)
-{
-    if (check_closed(server)) {
-        return;
-    }
-
-    purge_single_server(server, err, 0, NULL, REFRESH_ALWAYS);
-    lcb_maybe_breakout(server->instance);
-    start_errored_ctx(server, S_ERRDRAIN);
-}
-
-void
-mcserver_close(mc_SERVER *server)
-{
-    /* Should never be called twice */
-    lcb_assert(server->state != S_CLOSED);
-    start_errored_ctx(server, S_CLOSED);
-}
-
-/**
- * Call to signal an error or similar on the current socket.
- * @param server The server
- * @param next_state The next state (S_CLOSED or S_ERRDRAIN)
- */
-static void
-start_errored_ctx(mc_SERVER *server, mcserver_STATE next_state)
-{
-    lcbio_CTX *ctx = server->connctx;
-
-    server->state = next_state;
-    /* Cancel any pending connection attempt? */
-    lcbio_connreq_cancel(&server->connreq);
-
-    /* If the server is being destroyed, silence the timer */
-    if (next_state == S_CLOSED && server->io_timer != NULL) {
-        lcbio_timer_destroy(server->io_timer);
-        server->io_timer = NULL;
-    }
-
-    if (ctx == NULL) {
-        if (next_state == S_CLOSED) {
-            server_free(server);
-            return;
-        } else {
-            /* Not closed but don't have a current context */
-            server->pipeline.flush_start = (mcreq_flushstart_fn)server_connect;
-            if (mcserver_has_pending(server)) {
-                if (!lcbio_timer_armed(server->io_timer)) {
-                    /* TODO: Maybe throttle reconnection attempts? */
-                    lcbio_timer_rearm(server->io_timer, MCSERVER_TIMEOUT(server));
-                }
-                server_connect(server);
-            }
-        }
-
-    } else {
-        if (ctx->npending) {
-            /* Have pending items? */
-
-            /* Flush any remaining events */
-            lcbio_ctx_schedule(ctx);
-
-            /* Close the socket not to leak resources */
-            lcbio_shutdown(lcbio_ctx_sock(ctx));
-            if (next_state == S_ERRDRAIN) {
-                server->pipeline.flush_start = (mcreq_flushstart_fn)flush_errdrain;
-            }
-        } else {
-            finalize_errored_ctx(server);
-        }
-    }
-}
-
-/**
- * This function actually finalizes a ctx which has an error on it. If the
- * ctx has pending operations remaining then this function returns immediately.
- * Otherwise this will either reinitialize the connection or free the server
- * object depending on the actual object state (i.e. if it was closed or
- * simply errored).
- */
-static void
-finalize_errored_ctx(mc_SERVER *server)
-{
-    if (server->connctx->npending) {
-        return;
-    }
-
-    lcb_log(LOGARGS(server, DEBUG), LOGFMT "Finalizing ctx %p", LOGID(server), (void*)server->connctx);
-
-    /* Always close the existing context. */
-    lcbio_ctx_close(server->connctx, close_cb, NULL);
-    server->connctx = NULL;
-
-    /* And pretend to flush any outstanding data. There's nothing pending! */
-    release_unflushed_packets(server);
-
-    if (server->state == S_CLOSED) {
-        /* If the server is closed, time to free it */
-        server_free(server);
-    } else {
-        /* Otherwise, cycle the state back to CLEAN and reinit
-         * the connection */
-        server->state = S_CLEAN;
-        server->pipeline.flush_start = (mcreq_flushstart_fn)server_connect;
-        server_connect(server);
-    }
-}
-
-/**
- * This little function checks to see if the server struct is still valid, or
- * whether it should just be cleaned once no pending I/O remainds.
- *
- * If this function returns false then the server is still valid; otherwise it
- * is invalid and must not be used further.
- */
-static int
-check_closed(mc_SERVER *server)
-{
-    if (server->state == S_CLEAN) {
-        return 0;
-    }
-    lcb_log(LOGARGS(server, INFO), LOGFMT "Got handler after close. Checking pending calls (pending=%u)", LOGID(server), server->connctx->npending);
-    finalize_errored_ctx(server);
-    return 1;
-}