mt-libcouchbase 1.4.0

data/ext/libcouchbase/src/mc/forward.c

@@ -0,0 +1,186 @@
+/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *     Copyright 2014 Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#include "mcreq.h"
+#include "forward.h"
+#include "iovcursor-inl.h"
+#include <stdio.h>
+
+#define MINIMUM(a, b) (a) < (b) ? a : b
+
+static void
+span_from_first(mc_IOVCURSOR *cursor, unsigned size, nb_SPAN *span)
+{
+    nb_IOV dummy;
+    iovcursor_adv_first(cursor, size, &dummy);
+    CREATE_STANDALONE_SPAN(span, dummy.iov_base, dummy.iov_len);
+}
+
+void
+mc_iovinfo_init(mc_IOVINFO *info, const nb_IOV *iov, unsigned niov)
+{
+    unsigned ii;
+    info->c.iov = (void *)iov;
+    info->c.niov = niov;
+    info->c.offset = 0;
+    info->total = 0;
+
+    for (ii = 0; ii < niov; ii++) {
+        info->total += iov[ii].iov_len;
+    }
+}
+
+#define REQLEN_HDR(req) sizeof(req.bytes)
+#define REQLEN_
+
+lcb_error_t
+mc_forward_packet(mc_CMDQUEUE *cq,
+    mc_IOVINFO *info, mc_PACKET **pkt_p, mc_PIPELINE **pl_p, int options)
+{
+    /* stack based header with our modifications. this is copied into the
+     * packet's actual header */
+    protocol_binary_request_header hdr;
+    int vbid, srvix;
+    mc_IOVCURSOR *mincur = &info->c;
+
+    unsigned n_packet; /* total packet size */
+    unsigned n_header; /* extras + key + memcached header */
+    unsigned n_body_total; /* size of everything following the memcached header */
+    unsigned n_body_key; /* length of the key */
+    unsigned n_body_value; /* packetsize - hdrsize */
+
+    unsigned offset;
+
+    /* stack buffer and key pointer. stack buffer is used if the key is not
+     * contiguous */
+    char kbuf_s[256];
+    const char *kptr;
+
+    /* pipeline and packet for command */
+    mc_PIPELINE *pl;
+    mc_PACKET *pkt;
+    info->wanted = 0;
+
+    /* not enough bytes */
+    if (info->total < 24) {
+        info->wanted = 24;
+        return LCB_INCOMPLETE_PACKET;
+    }
+
+    iovcursor_peek(mincur, (char*)hdr.bytes, sizeof hdr.bytes, 0);
+
+    /* Initialize our size variables */
+    n_body_total = ntohl(hdr.request.bodylen);
+    n_body_key = ntohs(hdr.request.keylen);
+    n_header = sizeof hdr.bytes + n_body_key + hdr.request.extlen;
+    n_packet = n_body_total + sizeof hdr.bytes;
+    n_body_value = n_packet - n_header;
+
+    if (n_packet > info->total) {
+        info->wanted = n_packet;
+        return LCB_INCOMPLETE_PACKET;
+    }
+
+    info->total -= n_packet;
+
+    /* seek ahead to read the item's key into the header */
+    offset = sizeof hdr.bytes + hdr.request.extlen;
+
+    iovcursor_peek_ex(mincur, kbuf_s, &kptr, n_body_key, offset);
+
+    if (kptr == NULL) {
+        /* key is not contiguous? that's ok. use the static buffer */
+        kptr = kbuf_s;
+    }
+
+    if ((options & MC_FWD_OPT_NOMAP) == 0) {
+        lcbvb_map_key(cq->config, kptr, n_body_key, &vbid, &srvix);
+        if (srvix < 0 || (unsigned)srvix >= cq->npipelines) {
+            return LCB_NO_MATCHING_SERVER;
+        }
+        pl = cq->pipelines[srvix];
+        hdr.request.vbucket = htons(vbid);
+
+    } else {
+        pl = *pl_p;
+        if (!pl) {
+            return LCB_EINVAL;
+        }
+        srvix = pl->index;
+    }
+
+    pkt = mcreq_allocate_packet(pl);
+
+    if (pkt == NULL) {
+        return LCB_CLIENT_ENOMEM;
+    }
+
+    hdr.request.opaque = pkt->opaque;
+    pkt->extlen = hdr.request.extlen;
+    info->consumed = n_packet;
+
+    if (options & MC_FWD_OPT_COPY) {
+        /* reserve bytes for the entire packet */
+        mcreq_reserve_header(pl, pkt, n_header);
+        iovcursor_adv_copy(mincur, SPAN_BUFFER(&pkt->kh_span), n_header);
+        if (n_body_value) {
+            mcreq_reserve_value2(pl, pkt, n_body_value);
+            iovcursor_adv_copy(mincur,
+                SPAN_BUFFER(&pkt->u_value.single), n_body_value);
+            pkt->flags |= MCREQ_F_HASVALUE;
+        }
+
+    } else {
+        if (IOVCURSOR_HAS_CONTIG(mincur, n_header)) {
+            span_from_first(mincur, n_header, &pkt->kh_span);
+            pkt->flags |= MCREQ_F_KEY_NOCOPY;
+
+        } else {
+            /* header is fragmented into multiple IOVs */
+            mcreq_reserve_header(pl, pkt, n_header);
+            iovcursor_adv_copy(mincur, SPAN_BUFFER(&pkt->kh_span), n_header);
+        }
+
+        /* do we have a value payload still? */
+        if (n_body_value) {
+            pkt->flags |= MCREQ_F_HASVALUE | MCREQ_F_VALUE_NOCOPY;
+            if (IOVCURSOR_HAS_CONTIG(mincur, n_body_value)) {
+                span_from_first(mincur, n_body_value, &pkt->u_value.single);
+
+            } else {
+                /* body is fragmented */
+                iovcursor_adv_iovalloc(mincur, n_body_value,
+                    (nb_IOV**)&pkt->u_value.multi.iov,
+                    &pkt->u_value.multi.niov);
+                pkt->u_value.multi.total_length = n_body_value;
+                pkt->flags |= MCREQ_F_VALUE_IOV;
+            }
+        }
+    }
+
+    /* Copy the first 24 bytes into the header span */
+    memcpy(SPAN_BUFFER(&pkt->kh_span), hdr.bytes, sizeof hdr.bytes);
+
+    *pkt_p = pkt;
+    *pl_p = pl;
+
+    /* Set the UFWD flag. This causes the rest of the system to invoke the
+     * handler for the raw response, rather than the "Contiguous" structures*/
+    pkt->flags |= MCREQ_F_UFWD;
+    mcreq_sched_add(pl, pkt);
+    return LCB_SUCCESS;
+}

data/ext/libcouchbase/src/mc/forward.h

@@ -0,0 +1,90 @@
+/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *     Copyright 2014 Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#ifndef MC_FORWARD_H
+#define MC_FORWARD_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "iovcursor.h"
+
+
+/**
+ * Copy over the entire packet to the internal buffers. Input buffer is
+ * temporary.
+ */
+#define MC_FWD_OPT_COPY 0x01
+
+/**
+ * The server to send to is already set as `pl`. Don't perform vbucket mapping.
+ */
+#define MC_FWD_OPT_NOMAP 0x02
+
+void
+mc_iovinfo_init(mc_IOVINFO *info, const nb_IOV *iov, unsigned niov);
+
+/**
+ * Forward a packet to an upstream server.
+ * @param[in] cq the command queue
+ * @param[in] ctx a queue context used for scheduling/flushing the packets
+ * @param[in] info an 'IOVINFO' structure. See the structure documentation for
+ *        more details.
+ * @param[out] pkt a pointer to a packet, set to the resultant packet structure
+ * @param[out] pl the pipeline this packet is mapped to
+ * @param options Options modifying the behavior of this operation
+ * @return LCB_SUCCESS or an error code.
+ *
+ * Currently only some commands will successfully make sense to forward. This
+ * function only handles the lower level aspect of actually allocating or
+ * reserving the buffers required to forward the packet, but not actually
+ * handling the received data for the callbacks themselves.
+ *
+ * Note that this function does not currently "Collapse" consectutive IOV
+ * structures. Additionally the following should be noted:
+ *
+ * <ol>
+ * <li>
+ *     If the first IOV does not contain a contiguous buffer of the
+ *     { header, extras, key }, then it will be copied into a library-based
+ *     buffer.
+ * </li>
+ *
+ * <li>
+ *     If the total number of IOVs is greater than two, then niov-1 IOV
+ *     structures will be allocated via 'malloc'. This may not always happen
+ *     if the header _itself_ is fragmented
+ * </li>
+ *
+ * <li>The first 24 bytes of the header WILL BE MODIFIED by the library.
+ *     This will be used to modify the 'opaque' and 'vbucket' fields.
+ *     Take this into note if you need to keep track of their original values.
+ * </li>
+ *
+ * <li>Check the 'niov' counter to see if it is 0. If it's 0 then you should
+ *     make sure to reset the counter.
+ * </li>
+ * </ol>
+ */
+lcb_error_t
+mc_forward_packet(mc_CMDQUEUE *cq,
+    mc_IOVINFO *info, mc_PACKET **pkt, mc_PIPELINE **pl, int options);
+
+#ifdef __cplusplus
+}
+#endif
+#endif

data/ext/libcouchbase/src/mc/iovcursor-inl.h

@@ -0,0 +1,279 @@
+/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *     Copyright 2014 Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#include "iovcursor.h"
+#ifndef MINIMUM
+#define MINIMUM(a, b) (a) < (b) ? a : b
+#endif
+
+/**Status code returned by the peek_ex() function. Codes below 0 are errors,
+ * while codes above 0 are successes*/
+typedef enum {
+    /** Data would be fragmented and no `copytgt` parameter was provided */
+    IOVCURSOR_STATUS_FRAGMENTED = -2,
+    /** Pointer to data is referenced by the `contigptr` parameter */
+    IOVCURSOR_STATUS_CONTIGPTR_OK = 0,
+    /** Pointer to contiguous data cannot be obtained. It has been copied */
+    IOVCURSOR_STATUS_BUFCOPY_OK
+} iovcursor_STATUS;
+
+/**
+ * Obtain data from a cursor without advancing it
+ * @param cursor The cursor to read from
+ *
+ * @param copytgt A buffer in which to copy the data, in case a contiguous
+ * pointer to `size` bytes cannot be obtained.
+ *
+ * @param[out] contigref A pointer to be set to a contiguous region of memory
+ * containing `size` bytes
+ * @param size
+ * @param offset The offset at which to start reading
+ * @return A status code indicating success or failure of the operation
+ * @note Either `copytgt` or `contigref` _must_ be specified. It is ok to pass
+ * them both in the same function call (and the return status may be inspected
+ * to see which one of the parameters contains the actual data).
+ */
+static iovcursor_STATUS
+iovcursor_peek_ex(const mc_IOVCURSOR *cursor,
+    char *copytgt, const char **contigref,
+    unsigned size, unsigned offset)
+{
+    unsigned ii;
+    const nb_IOV *iov = cursor->iov;
+    offset += cursor->offset;
+    for (ii = 0; ii < cursor->niov && size > 0; ++ii) {
+        unsigned contiglen, tmpoff;
+        const char *srcbuf;
+        const nb_IOV *cur = iov + ii;
+
+        if (offset) {
+            if (offset >= cur->iov_len) {
+                offset -= cur->iov_len;
+                continue;
+            } else {
+                tmpoff = offset;
+                offset = 0;
+            }
+        } else {
+            tmpoff = 0;
+        }
+
+        contiglen = cur->iov_len - tmpoff;
+        srcbuf = (const char *)cur->iov_base + tmpoff;
+
+        /* We always end up returning from these following blocks and _not_
+         * the end of the loop:
+         *
+         * For contiguous buffers, contiglen >= size is true on the first
+         * iteration.
+         *
+         * For fragmented buffers, contiglen >= size is _eventually_ true.
+         * however during the first initial IOVs, the buffer is fragmented
+         * but as the output is copied the size variable is decremented. Note
+         * that during this process we also set contigref to NULL to avoid
+         * accidentally thinking that the full data set is contiguous
+         */
+        if (size <= contiglen) {
+            if (contigref) {
+                *contigref = srcbuf;
+                return IOVCURSOR_STATUS_CONTIGPTR_OK;
+            } else {
+                memcpy(copytgt, srcbuf, size);
+                return IOVCURSOR_STATUS_BUFCOPY_OK;
+            }
+        } else if (copytgt == NULL) {
+            *contigref = NULL;
+            return IOVCURSOR_STATUS_FRAGMENTED;
+        } else {
+            /* copy and continue */
+            unsigned to_copy = MINIMUM(size, cur->iov_len - tmpoff);
+            memcpy(copytgt, srcbuf, to_copy);
+            copytgt += to_copy;
+            if (contigref) {
+                *contigref = NULL;
+                contigref = NULL;
+            }
+
+            /* We've copied, so ignore the 'contigref' */
+            contigref = NULL;
+            if (!(size -= to_copy)) {
+                return IOVCURSOR_STATUS_BUFCOPY_OK;
+            }
+        }
+    }
+
+    assert(!size);
+    *contigref = NULL;
+    return IOVCURSOR_STATUS_FRAGMENTED;
+}
+
+/**
+ * Copy data to the target buffer, without modifying the offset
+ * @param cursor The cursror to read from
+ * @param buf The target buffer
+ * @param size The number of bytes to copy
+ * @param offset Position in the input at which to start copying
+ * @return true if there were sufficient bytes to copy, false otherwise.
+ */
+static int
+iovcursor_peek(const mc_IOVCURSOR *cursor, char *buf,
+    unsigned size, unsigned offset)
+{
+    int rv = iovcursor_peek_ex(cursor, buf, NULL, size, offset);
+    return rv == IOVCURSOR_STATUS_BUFCOPY_OK;
+}
+
+/**
+ * Populate an IOV structure with the effective offset of the first IOV contained
+ * within the cursor
+ * @param cursor
+ * @param maxsize Maximum length of resultant IOV, if the cursor's first IOV
+ * length is greated
+ * @param iov The iov to be initialized
+ */
+static unsigned
+iovcursor_adv_first(mc_IOVCURSOR *cursor, unsigned maxsize, nb_IOV *iov)
+{
+    const char *srcbuf = (const char *)cursor->iov->iov_base + cursor->offset;
+
+    /* Set the target */
+    iov->iov_base = (void*)srcbuf;
+    iov->iov_len = MINIMUM(cursor->iov->iov_len - cursor->offset, maxsize);
+
+    if (iov->iov_len == (cursor->iov->iov_len - cursor->offset)) {
+        /* did we swallow the entire source iov? - consume it */
+        cursor->iov++;
+        cursor->niov--;
+        cursor->offset = 0;
+    } else {
+        /* increase the offset */
+        cursor->offset += iov->iov_len;
+    }
+
+    return iov->iov_len;
+}
+
+/**
+ * Copy data to the target buffer, advancing the cursor
+ * @param cursor The cursor
+ * @param tgt The buffer to copy to
+ * @param size The number of bytes to copy
+ *
+ * @warning No check is made to see if `size` is greater than the amounf of
+ * data contained within the cursor. Exceeding the amount of data will result
+ * in undefined behavior.
+ */
+static void
+iovcursor_adv_copy(mc_IOVCURSOR *cursor, char *tgt, unsigned size)
+{
+    nb_IOV tmpiov;
+    nb_IOV *iov;
+    unsigned niov;
+
+    size -= iovcursor_adv_first(cursor, size, &tmpiov);
+    memcpy(tgt, tmpiov.iov_base, tmpiov.iov_len);
+    tgt += tmpiov.iov_len;
+
+    /* assign iov and iov now, since adv_first() may have modified them */
+    iov = cursor->iov;
+    niov = cursor->niov;
+
+    while (size) {
+        unsigned to_copy = MINIMUM(iov->iov_len, size);
+        const char *srcbuf = (const char *)iov->iov_base;
+        memcpy(tgt, srcbuf, to_copy);
+        tgt += to_copy;
+        size -= to_copy;
+
+        if (to_copy != iov->iov_len) {
+            cursor->offset = to_copy;
+            assert(!size);
+            break;
+        }
+
+        iov++;
+        niov--;
+    }
+
+    /* modify the variables */
+    cursor->iov = iov;
+    cursor->niov = niov;
+}
+
+/**
+ * Macro which determines if the specific cursor's first IOV has enough
+ * data to ensure a contiguous memory region of a specific amount of bytes
+ * @param mincur the cursor
+ * @param n the number of bytes to check for
+ * @return nonzero if the requested size is available
+ */
+#define IOVCURSOR_HAS_CONTIG(mincur, n) \
+    ((mincur)->iov->iov_len - (mincur)->offset) >= n
+
+/**
+ * Create an allocated array of IOVs which point to a subset of IOVs within
+ * the current cursor. This function will also advance the cursor position.
+ *
+ * @param cursor The cursor to get the offsets from
+ * @param size The size the resultant array should cover
+ * @param[out] arr The IOV array which shall contain the offsets. The array
+ * should be freed by free() when no longer required.
+ * @param[out] narr Number of elements in the resultant array.
+ */
+static void
+iovcursor_adv_iovalloc(mc_IOVCURSOR *cursor, unsigned size,
+    nb_IOV **p_arr, unsigned *p_narr)
+{
+    unsigned ii, narr;
+    nb_IOV dummy, *arr;
+
+    /* chop off the first IOV for convenience */
+    size -= iovcursor_adv_first(cursor, size, &dummy);
+    narr = 1;
+
+    if (size) {
+        unsigned cursz = size;
+        for (ii = 0; cursz > 0; ++ii) {
+            cursz -= MINIMUM(cursz, cursor->iov[ii].iov_len);
+        }
+        narr += ii;
+    }
+
+    arr = (nb_IOV*) malloc(sizeof(*arr) * narr);
+    arr[0] = dummy;
+
+    for (ii = 1; size > 0; ++ii) {
+        unsigned to_adv = MINIMUM(size, cursor->iov->iov_len);
+        const char *srcbuf = (const char*)cursor->iov->iov_base;
+
+        arr[ii].iov_base = (void*)srcbuf;
+        arr[ii].iov_len = MINIMUM(size, cursor->iov->iov_len);
+
+        size -= to_adv;
+
+        if (size == 0 && to_adv < cursor->iov->iov_len) {
+            /* did we not copy the entire iov? */
+            cursor->offset = to_adv;
+        } else {
+            cursor->iov++;
+            cursor->niov--;
+        }
+    }
+
+    *p_arr = arr;
+    *p_narr = narr;
+}

data/ext/libcouchbase/src/mc/iovcursor.h

@@ -0,0 +1,66 @@
+/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *     Copyright 2014 Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#ifndef MC_IOVCURSOR_H
+#define MC_IOVCURSOR_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/** Minimal cursor */
+typedef struct {
+    /**The IOV array containing the buffer offsets. This is initialized to the
+     * first element of the array on input. As data is consumed by the
+     * library, this pointer value will increment.*/
+    nb_IOV *iov;
+
+    /**Number of elements in the IOV array. This is decremented as the `iov`
+     * field is incremented.*/
+    unsigned niov;
+
+    /**Offset into first IOV structure which contains data. This is used
+     * if the IOV contains partially consumed data. The library sets this
+     * field if a packet ends in the middle of an IOV buffer*/
+    unsigned offset;
+} mc_IOVCURSOR;
+
+typedef struct {
+    /** Cursor element */
+    mc_IOVCURSOR c;
+
+    /**The total number of bytes used by the library in the last packet
+     * successfuly processed.*/
+    unsigned consumed;
+
+    /**Number of bytes wanted for next operation (OUT). This contains the
+     * total number of bytes (including any within the buffer already).
+     * The library does not read from this value. */
+    unsigned wanted;
+
+    /**The total amount of data within the IOV buffers. This is initialized
+     * in the mc_iovinfo_init() function by traversing through all the elements
+     * and adding their `iov_len` fields. If using the `IOVINFO` structure
+     * in a read loop, you will want to increment this whenever new data has
+     * been placed into buffers*/
+    unsigned total;
+} mc_IOVINFO;
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif