libcouchbase 1.2.8 → 1.3.0

data/ext/libcouchbase/src/tracing/threshold_logging_tracer.cc

@@ -0,0 +1,171 @@
+/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *     Copyright 2018 Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#include "internal.h"
+
+#include <iostream>
+#include <vector>
+
+#define LOGARGS(tracer, lvl) tracer->m_settings, "tracer", LCB_LOG_##lvl, __FILE__, __LINE__
+
+using namespace lcb::trace;
+
+LIBCOUCHBASE_API lcbtrace_TRACER *lcbtrace_new(lcb_t instance, uint64_t flags)
+{
+    if (instance == NULL || (flags & LCBTRACE_F_THRESHOLD) == 0) {
+        return NULL;
+    }
+    return (new ThresholdLoggingTracer(instance))->wrap();
+}
+
+extern "C" {
+static void tlt_destructor(lcbtrace_TRACER *wrapper)
+{
+    if (wrapper == NULL) {
+        return;
+    }
+    if (wrapper->cookie) {
+        ThresholdLoggingTracer *tracer = reinterpret_cast< ThresholdLoggingTracer * >(wrapper->cookie);
+        delete tracer;
+        wrapper->cookie = NULL;
+    }
+    delete wrapper;
+}
+
+static void tlt_report(lcbtrace_TRACER *wrapper, lcbtrace_SPAN *span)
+{
+    if (wrapper == NULL || wrapper->cookie == NULL) {
+        return;
+    }
+
+    ThresholdLoggingTracer *tracer = reinterpret_cast< ThresholdLoggingTracer * >(wrapper->cookie);
+    if (span->tags[LCBTRACE_TAG_SERVICE] == LCBTRACE_TAG_SERVICE_KV) {
+        if (lcbtrace_span_is_orphaned(span)) {
+            tracer->add_orphan(span);
+        } else {
+            tracer->check_threshold(span);
+        }
+    }
+}
+}
+
+lcbtrace_TRACER *ThresholdLoggingTracer::wrap()
+{
+    if (m_wrapper) {
+        return m_wrapper;
+    }
+    m_wrapper = new lcbtrace_TRACER();
+    m_wrapper->version = 0;
+    m_wrapper->flags = 0;
+    m_wrapper->cookie = this;
+    m_wrapper->destructor = tlt_destructor;
+    m_wrapper->v.v0.report = tlt_report;
+    return m_wrapper;
+}
+
+ReportedSpan ThresholdLoggingTracer::convert(lcbtrace_SPAN *span)
+{
+    ReportedSpan orphan;
+    orphan.duration = span->duration();
+    Json::Value entry;
+    entry["operation_name"] = span->m_opname;
+    entry["operation_id"] = span->tags[LCBTRACE_TAG_OPERATION_ID];
+    entry["last_local_id"] = span->tags[LCBTRACE_TAG_LOCAL_ID];
+    entry["last_local_address"] = span->tags[LCBTRACE_TAG_LOCAL_ADDRESS];
+    entry["last_remote_address"] = span->tags[LCBTRACE_TAG_PEER_ADDRESS];
+    entry["server_us"] = span->tags[LCBTRACE_TAG_PEER_LATENCY];
+    entry["total_us"] = (Json::UInt64)orphan.duration;
+    orphan.payload = Json::FastWriter().write(entry);
+    return orphan;
+}
+
+void ThresholdLoggingTracer::add_orphan(lcbtrace_SPAN *span)
+{
+    m_orphans.push(convert(span));
+}
+
+void ThresholdLoggingTracer::check_threshold(lcbtrace_SPAN *span)
+{
+    if (span->duration() > m_settings->tracer_threshold[LCBTRACE_THRESHOLD_KV]) {
+        m_threshold.push(convert(span));
+    }
+}
+
+void ThresholdLoggingTracer::flush_queue(FixedQueue< ReportedSpan > &queue, const char *message)
+{
+    std::vector< ReportedSpan > &slice = queue.get_sorted();
+    Json::Value entries;
+    entries["service"] = "kv";
+    entries["count"] = (Json::UInt)slice.size();
+    Json::Value top;
+    for (size_t ii = 0; ii < slice.size(); ii++) {
+        Json::Value entry;
+        if (Json::Reader().parse(slice[ii].payload, entry)) {
+            top.append(entry);
+        }
+    }
+    entries["top"] = top;
+    std::string doc = Json::FastWriter().write(entries);
+    if (doc.size() > 0 && doc[doc.size() - 1] == '\n') {
+        doc[doc.size() - 1] = '\0';
+    }
+    lcb_log(LOGARGS(this, WARN), "%s: %s", message, doc.c_str());
+    queue.clear();
+}
+
+void ThresholdLoggingTracer::flush_orphans()
+{
+    lcb_U32 tv = m_settings->tracer_orphaned_queue_flush_interval;
+    if (tv == 0) {
+        m_oflush.cancel();
+    } else {
+        m_oflush.rearm(tv);
+    }
+    if (m_orphans.empty()) {
+        return;
+    }
+    flush_queue(m_orphans, "Orphan responses observed");
+}
+
+void ThresholdLoggingTracer::flush_threshold()
+{
+    lcb_U32 tv = m_settings->tracer_threshold_queue_flush_interval;
+    if (tv == 0) {
+        m_tflush.cancel();
+    } else {
+        m_tflush.rearm(tv);
+    }
+    if (m_threshold.empty()) {
+        return;
+    }
+    flush_queue(m_threshold, "Operations over threshold");
+}
+
+ThresholdLoggingTracer::ThresholdLoggingTracer(lcb_t instance)
+    : m_wrapper(NULL), m_settings(instance->settings), m_orphans(LCBT_SETTING(instance, tracer_orphaned_queue_size)),
+      m_threshold(LCBT_SETTING(instance, tracer_threshold_queue_size)), m_oflush(instance->iotable, this),
+      m_tflush(instance->iotable, this)
+{
+    lcb_U32 tv = m_settings->tracer_orphaned_queue_flush_interval;
+    if (tv > 0) {
+        m_oflush.rearm(tv);
+    }
+    tv = m_settings->tracer_threshold_queue_flush_interval;
+    if (tv > 0) {
+        m_tflush.rearm(tv);
+    }
+}

data/ext/libcouchbase/src/tracing/tracer.cc

@@ -0,0 +1,53 @@
+/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *     Copyright 2017 Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#include "internal.h"
+
+using namespace lcb::trace;
+
+LIBCOUCHBASE_API void lcbtrace_destroy(lcbtrace_TRACER *tracer)
+{
+    if (tracer && tracer->destructor) {
+        tracer->destructor(tracer);
+    }
+}
+
+LIBCOUCHBASE_API
+lcbtrace_SPAN *lcbtrace_span_start(lcbtrace_TRACER *tracer, const char *opname, uint64_t start, lcbtrace_REF *ref)
+{
+    lcbtrace_REF_TYPE type = LCBTRACE_REF_NONE;
+    lcbtrace_SPAN *other = NULL;
+    if (ref) {
+        type = ref->type;
+        other = ref->span;
+    }
+    return new Span(tracer, opname, start, type, other);
+}
+
+LIBCOUCHBASE_API
+lcbtrace_TRACER *lcb_get_tracer(lcb_t instance)
+{
+    return (instance && instance->settings) ? instance->settings->tracer : NULL;
+}
+
+LIBCOUCHBASE_API
+void lcb_set_tracer(lcb_t instance, lcbtrace_TRACER *tracer)
+{
+    if (instance && instance->settings) {
+        instance->settings->tracer = tracer;
+    }
+}

data/ext/libcouchbase/src/tracing/tracing-internal.h

@@ -0,0 +1,213 @@
+/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *     Copyright 2018 Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#ifndef LCB_TRACING_INTERNAL_H
+#define LCB_TRACING_INTERNAL_H
+
+#ifdef LCB_TRACING
+#include <libcouchbase/tracing.h>
+#include "rnd.h"
+
+#ifdef __cplusplus
+
+#include <queue>
+
+namespace lcb
+{
+namespace trace
+{
+
+class Span
+{
+  public:
+    Span(lcbtrace_TRACER *tracer, const char *opname, uint64_t start, lcbtrace_REF_TYPE ref, lcbtrace_SPAN *other);
+
+    void finish(uint64_t finish);
+    uint64_t duration()
+    {
+        return m_finish - m_start;
+    }
+
+    template < typename T > void add_tag(const char *name, T value);
+
+    lcbtrace_TRACER *m_tracer;
+    std::string m_opname;
+    uint64_t m_span_id;
+    uint64_t m_start;
+    uint64_t m_finish;
+    bool m_orphaned;
+    Json::Value tags;
+    Span *m_parent;
+};
+
+struct ReportedSpan {
+    uint64_t duration;
+    std::string payload;
+
+    bool operator<(const ReportedSpan &rhs) const
+    {
+        return duration < rhs.duration;
+    }
+};
+
+template < typename T > class FixedQueue
+{
+  public:
+    explicit FixedQueue(size_t capacity) : m_capacity(capacity) {}
+
+    void push(T item)
+    {
+        if (m_items.size() < m_capacity) {
+            m_items.push_back(item);
+            std::push_heap(m_items.begin(), m_items.end());
+        } else {
+            std::sort_heap(m_items.begin(), m_items.end());
+            if (m_items.front() < item) {
+                m_items[0] = item;
+            }
+            std::make_heap(m_items.begin(), m_items.end());
+        }
+    }
+
+    size_t size()
+    {
+        return m_items.size();
+    }
+
+    bool empty()
+    {
+        return m_items.empty();
+    }
+
+    void clear()
+    {
+        m_items.clear();
+    }
+
+    std::vector< T > &get_sorted()
+    {
+        std::sort_heap(m_items.begin(), m_items.end());
+        return m_items;
+    }
+
+  private:
+    size_t m_capacity;
+    std::vector< T > m_items;
+};
+
+class ThresholdLoggingTracer
+{
+    lcbtrace_TRACER *m_wrapper;
+    lcb_settings *m_settings;
+
+    FixedQueue< ReportedSpan > m_orphans;
+    FixedQueue< ReportedSpan > m_threshold;
+
+    void flush_queue(FixedQueue< ReportedSpan > &queue, const char *message);
+    ReportedSpan convert(lcbtrace_SPAN *span);
+
+  public:
+    ThresholdLoggingTracer(lcb_t instance);
+
+    lcbtrace_TRACER *wrap();
+    void add_orphan(lcbtrace_SPAN *span);
+    void check_threshold(lcbtrace_SPAN *span);
+
+    void flush_orphans();
+    void flush_threshold();
+
+    lcb::io::Timer< ThresholdLoggingTracer, &ThresholdLoggingTracer::flush_orphans > m_oflush;
+    lcb::io::Timer< ThresholdLoggingTracer, &ThresholdLoggingTracer::flush_threshold > m_tflush;
+};
+
+} // namespace trace
+} // namespace lcb
+
+extern "C" {
+#endif
+LCB_INTERNAL_API
+void lcbtrace_span_add_system_tags(lcbtrace_SPAN *span, lcb_settings *settings, const char *service);
+LCB_INTERNAL_API
+void lcbtrace_span_set_parent(lcbtrace_SPAN *span, lcbtrace_SPAN *parent);
+LCB_INTERNAL_API
+void lcbtrace_span_set_orphaned(lcbtrace_SPAN *span, int val);
+
+#define LCBTRACE_KV_START(settings, cmd, operation_name, opaque, outspan)                                              \
+    if ((settings)->tracer) {                                                                                          \
+        lcbtrace_REF ref;                                                                                              \
+        char opid[20] = {};                                                                                            \
+        snprintf(opid, sizeof(opid), "0x%x", (int)opaque);                                                             \
+        ref.type = LCBTRACE_REF_CHILD_OF;                                                                              \
+        ref.span = (cmd->_hashkey.type & LCB_KV_TRACESPAN) ? (lcbtrace_SPAN *)cmd->_hashkey.contig.bytes : NULL;       \
+        outspan = lcbtrace_span_start((settings)->tracer, operation_name, LCBTRACE_NOW, &ref);                         \
+        lcbtrace_span_add_tag_str(outspan, LCBTRACE_TAG_OPERATION_ID, opid);                                           \
+        lcbtrace_span_add_system_tags(outspan, (settings), LCBTRACE_TAG_SERVICE_KV);                                   \
+    }
+
+#define LCBTRACE_KV_COMPLETE(pipeline, request, response)                                                              \
+    do {                                                                                                               \
+        lcbtrace_SPAN *span = MCREQ_PKT_RDATA(request)->span;                                                          \
+        if (span) {                                                                                                    \
+            lcbtrace_span_add_tag_uint64(span, LCBTRACE_TAG_PEER_LATENCY, (response)->duration());                     \
+            lcb::Server *server = static_cast< lcb::Server * >(pipeline);                                              \
+            const lcb_host_t &remote = server->get_host();                                                             \
+            std::string hh;                                                                                            \
+            if (remote.ipv6) {                                                                                         \
+                hh.append("[").append(remote.host).append("]:").append(remote.port);                                   \
+            } else {                                                                                                   \
+                hh.append(remote.host).append(":").append(remote.port);                                                \
+            }                                                                                                          \
+            lcbtrace_span_add_tag_str(span, LCBTRACE_TAG_PEER_ADDRESS, hh.c_str());                                    \
+            lcbio_CTX *ctx = server->connctx;                                                                          \
+            if (ctx) {                                                                                                 \
+                char local_id[34] = {};                                                                                \
+                snprintf(local_id, sizeof(local_id), "%016" PRIx64 "/%016" PRIx64,                                     \
+                         (lcb_U64)server->get_settings()->iid, ctx->sock->id);                                         \
+                lcbtrace_span_add_tag_str(span, LCBTRACE_TAG_LOCAL_ID, local_id);                                      \
+                lcbtrace_span_add_tag_str(span, LCBTRACE_TAG_LOCAL_ADDRESS,                                            \
+                                          lcbio__inet_ntop(&ctx->sock->info->sa_local).c_str());                       \
+            }                                                                                                          \
+        }                                                                                                              \
+    } while (0);
+
+#define LCBTRACE_KV_CLOSE(request)                                                                                     \
+    do {                                                                                                               \
+        lcbtrace_SPAN *span = MCREQ_PKT_RDATA(request)->span;                                                          \
+        if (span) {                                                                                                    \
+            lcbtrace_span_finish(span, LCBTRACE_NOW);                                                                  \
+            MCREQ_PKT_RDATA(request)->span = NULL;                                                                     \
+        }                                                                                                              \
+    } while (0);
+
+#define LCBTRACE_KV_FINISH(pipeline, request, response)                                                                \
+    LCBTRACE_KV_COMPLETE(pipeline, request, response)                                                                  \
+    LCBTRACE_KV_CLOSE(request)
+
+#ifdef __cplusplus
+}
+#endif
+
+#else
+
+#define LCBTRACE_KV_START(settings, cmd, operation_name, opaque, outspan)
+#define LCBTRACE_KV_COMPLETE(pipeline, request, response)
+#define LCBTRACE_KV_CLOSE(request)
+#define LCBTRACE_KV_FINISH(pipeline, request, response)
+
+#endif /* LCB_TRACING */
+
+#endif /* LCB_TRACING_INTERNAL_H */

data/ext/libcouchbase/src/utilities.c

@@ -34,6 +34,11 @@ extern lcb_uint64_t lcb_byteswap64(lcb_uint64_t val)
     return ret;
 }
 
+extern lcb_uint16_t lcb_byteswap16(lcb_uint16_t val)
+{
+    return ((val & 0xff) << 8) | ((val >> 8) & 0xff);
+}
+
 /**
  * While the C standard library uses 'putenv' for environment variable
  * manipulation, POSIX defines setenv (which works sanely) but Windows