mt-libcouchbase 1.4.0

data/ext/libcouchbase/tools/docgen/placeholders.h

@@ -0,0 +1,211 @@
+/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *     Copyright 2015 Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#ifndef CBC_PILLOWFIGHT_DOCGEN_H
+#define CBC_PILLOWFIGHT_DOCGEN_H
+
+#include <string>
+#include <vector>
+#include <algorithm>
+#include "seqgen.h"
+#include "loc.h"
+
+namespace Pillowfight {
+namespace Placeholders {
+
+/**
+ * Placeholder specification.
+ * This contains information about a single placeholder's input specification.
+ */
+class Spec {
+public:
+    /**
+     * @param s placeholder string
+     * @param minval_ minimum value for replacement
+     * @param maxval_ maxmimum value for replacement
+     * @param sequential_ if replacement should be done sequentially
+     */
+    Spec(const std::string &s, unsigned minval, unsigned maxval, unsigned sequential)
+    : m_term(s), m_minval(minval), m_maxval(maxval), m_sequential(sequential) {
+    }
+
+    SeqGenerator *createSeqgen(int total, int cur) const {
+        if (m_sequential) {
+            return new SeqGenerator(m_minval, m_maxval, total, cur);
+        } else {
+            return new SeqGenerator(m_minval, m_maxval);
+        }
+    }
+
+    const std::string& term() const { return m_term; }
+
+private:
+    std::string m_term; // Placeholder string to search for
+    unsigned m_minval;
+    unsigned m_maxval;
+    bool m_sequential;
+};
+
+/**
+ * Mapping of a placeholder specification as it relates to a given template
+ * document.
+ *
+ * This class can be copied since we don't have pointer references within the
+ * match itself.
+ */
+class Match {
+public:
+    // Size of the placeholder (derived from spec)
+    size_t size() const { return m_placeholder->term().size(); }
+    size_t offset() const { return m_offset; }
+    const Spec& spec() const { return *m_placeholder; }
+
+    template <class Tin, class Tres> static void
+    find(const std::string& base, const Tin& specs, Tres& results) {
+
+        for (size_t ii = 0; ii < specs.size(); ii++) {
+            const Spec& pl = specs[ii];
+            size_t findpos = base.find(pl.term());
+            if (findpos != std::string::npos) {
+                results.push_back(Match(&pl, findpos));
+            }
+        }
+        std::sort(results.begin(), results.end(), compare);
+    }
+
+private:
+    // Actual placeholder containing the information
+    const Spec *m_placeholder;
+
+    // Offset into the document in which the placeholder text begins
+    size_t m_offset;
+    Match(const Spec *spec_, size_t off)
+    : m_placeholder(spec_), m_offset(off) {
+    }
+
+    static bool compare(const Match& a, const Match& b) {
+        return a.m_offset < b.m_offset;
+    }
+};
+
+/**
+ * A document with its relevant placeholders. This contains the (constant)
+ * document string along with its placeholder information.
+ *
+ * The document is split into fragments, where some fragments are constant
+ * and some are empty and have DocPlaceholder objects mapped to them.
+ */
+class DocumentMatches {
+public:
+    DocumentMatches(const std::string& original,
+        const std::vector<Spec>& placeholders) : m_base(original) {
+
+        Match::find(m_base, placeholders, m_matches);
+
+        const Loc baseloc(m_base.c_str(), m_base.size());
+        m_fragments.push_back(baseloc);
+
+        for (size_t ii = 0; ii < m_matches.size(); ii++) {
+            Match& dph = m_matches[ii];
+
+            // Location of text to cut out
+            Loc dph_loc(m_base.c_str() + dph.offset(), dph.size());
+
+            // Make the last fragment end at the beginning of the current
+            // placeholder
+            m_fragments.back().rtrim_to(dph_loc);
+
+            // Add the replacement index (current index)
+            m_matchix_to_fragix.push_back(m_fragments.size());
+
+            // Add the empty fragment as a Loc to represent the placeholder
+            m_fragments.push_back(Loc());
+
+            // Make the next segment contain the rest of the document.
+            // If there are more placeholders, then this fragment is truncated
+            Loc next_seg;
+            next_seg.begin_at_end(baseloc, dph_loc, Loc::NO_OVERLAP);
+            m_fragments.push_back(next_seg);
+        }
+    }
+
+    const std::vector<Match>& matches() const { return m_matches; }
+
+private:
+    DocumentMatches(const DocumentMatches& other);
+
+    friend class Substitutions;
+    std::string m_base; // Base document text
+    std::vector<Loc> m_fragments; // Fragments of the document
+    std::vector<Match> m_matches;
+    std::vector<int> m_matchix_to_fragix; // Mapping of matches to IOV indexes
+};
+
+class Substitutions {
+public:
+    // Backing buffer for data
+    typedef std::vector<std::string> Backbuffer;
+
+    Substitutions(const DocumentMatches *matches, int total, int cur)
+    : m_matches(matches) {
+        for (size_t ii = 0; ii < m_matches->m_matches.size(); ii++) {
+            const Match& m = m_matches->m_matches[ii];
+            SeqGenerator *gen = m.spec().createSeqgen(total, cur);
+            m_generators.resize(ii + 1);
+            m_generators[ii] = gen;
+        }
+        for (size_t ii = 0; ii < m_matches->m_fragments.size(); ii++) {
+            m_iovs.push_back(m_matches->m_fragments[ii].to_iov());
+        }
+    }
+
+    /**
+     * Create the IOVs necessary for assembling the document
+     * @param[out] iovs Vector to contain the output IOVs
+     * @param[out] backbuf backing buffers for substitutions
+     */
+    void makeIovs(std::vector<lcb_IOV>& iovs, Backbuffer& backbuf) {
+        iovs = m_iovs;
+        backbuf.resize(m_matches->matches().size());
+
+        for (size_t ii = 0; ii < m_matches->matches().size(); ii++) {
+            char buf[64];
+            std::string& output_str = backbuf[ii];
+            lcb_IOV& output_iov = iovs[m_matches->m_matchix_to_fragix[ii]];
+            SeqGenerator *gen = m_generators[ii];
+
+            // Get the number
+            uint32_t cur = gen->next();
+
+            sprintf(buf, "%u", cur);
+            output_str.assign(buf);
+            output_iov.iov_base = const_cast<char *>(output_str.c_str());
+            output_iov.iov_len = output_str.size();
+        }
+    }
+
+private:
+    const DocumentMatches *m_matches;
+    // Array of generators, one for each Match index
+    std::vector<SeqGenerator *> m_generators;
+    std::vector<lcb_IOV> m_iovs;
+};
+
+} // namespace
+} // namespace
+
+#endif

data/ext/libcouchbase/tools/docgen/seqgen.h

@@ -0,0 +1,122 @@
+/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *     Copyright 2016 Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#ifndef CBC_PILLOWFIGHT_SEQGEN_H
+#define CBC_PILLOWFIGHT_SEQGEN_H
+
+namespace Pillowfight {
+
+/**
+ * Stateful sequence generator, divides sequences based on number of threads.
+ *
+ * On input, it takes a total range amount as well as a number of threads
+ * to serve the range, and finally, the index of the current thread.
+ *
+ * There is one generator per thread, as it is stateful
+ */
+class SeqGenerator {
+public:
+    /** Constructor for sequential key generators */
+    SeqGenerator(uint32_t start, uint32_t end, int num_workers, int cur_worker) {
+        offset = start;
+        uint32_t total = end - start;
+        total_self = total / num_workers;
+        locked = std::vector<bool>(total_self, false);
+        lnum = 0;
+        offset += total_self * cur_worker;
+        rnum = 0;
+        sequential = true;
+        curr_seqno = 0;
+    }
+
+    /** Initialize as a random range */
+    SeqGenerator(uint32_t start, uint32_t end) {
+        total_self = end - start;
+        locked = std::vector<bool>(total_self, false);
+        lnum = 0;
+        offset = start;
+        rnum = 0;
+        curr_seqno = 0;
+        sequential = false;
+
+        for (int ii = 0; ii < 8192; ii++) {
+            seqpool.push_back(rand());
+        }
+    }
+
+    /**
+     * Get the next sequence in range
+     * @return A number appropriate for the current sequence
+     */
+    uint32_t next() {
+        if (sequential) {
+            rnum++;
+            rnum %= total_self;
+            rnum += offset;
+            return rnum;
+
+        } else {
+            rnum += seqpool[curr_seqno];
+            curr_seqno++;
+            if (curr_seqno >= seqpool.size()) {
+                curr_seqno = 0;
+            }
+            uint32_t seq = rnum;
+            seq %= total_self;
+            seq += offset;
+            return seq;
+        }
+    }
+
+    uint32_t checkout() {
+        uint32_t num;
+        num = next();
+        if (lnum == locked.size()) {
+            lnum = 0;
+            std::fill(locked.begin(), locked.end(), false);
+        } else {
+            while (locked[num - offset]) {
+                num = next();
+            }
+        }
+        locked[num - offset] = true;
+        lnum++;
+        return num;
+    }
+
+    void checkin(uint32_t num) {
+        if (num - offset < locked.size()) {
+            locked[num - offset] = 0;
+        }
+    }
+
+    uint32_t maxItems() const {
+        return total_self;
+    }
+
+private:
+    bool sequential;
+    std::vector<uint32_t> seqpool;
+    std::vector<bool> locked; // lock markers
+    uint32_t lnum; // number of locked keys
+    uint32_t rnum; // internal iterator
+    uint32_t offset; // beginning numerical offset
+    uint32_t total_self; // maximum value of iterator
+    uint32_t curr_seqno;
+};
+}
+#endif

data/ext/libcouchbase/tools/extract-packets.rb

@@ -0,0 +1,110 @@
+#!/usr/bin/ruby
+
+# This tool extracts packet dumps from the TRACE log statements
+# when the library compiled with -DLCB_DUMP_PACKETS=ON
+
+require 'stringio'
+require 'base64'
+
+def pad(dir, *lines)
+  lines.flatten.join("\n").gsub(/^/, dir == 'snd' ? '> ' : '< ')
+end
+
+def format_bytes(buf, style = :wide)
+  out = StringIO.new
+  width = style == :wide ? 32 : 16
+  full_rows = buf.size / width
+  remainder = buf.size % width
+
+  if style == :wide
+    out.print("         +-------------------------------------------------------------------------------------------------+\n" \
+              "         |  0  1  2  3  4  5  6  7  8  9  a  b  c  d  e  f  0  1  2  3  4  5  6  7  8  9  a  b  c  d  e  f |\n" \
+              "+--------+-------------------------------------------------------------------------------------------------+--------------------------------+")
+  else
+    out.print("         +-------------------------------------------------+\n" \
+              "         |  0  1  2  3  4  5  6  7  8  9  a  b  c  d  e  f |\n" \
+              "+--------+-------------------------------------------------+----------------+")
+  end
+
+  row = 0
+  while row < full_rows
+    row_start_index = row * width
+    # prefix
+    out.printf("\n|%08x|", row_start_index)
+    row_end_index = row_start_index + width
+    # hex
+    i = row_start_index
+    while i < row_end_index
+      out.printf(" %02x", buf[i].ord)
+      i += 1
+    end
+    out.printf(" |")
+    # ascii
+    i = row_start_index
+    while i < row_end_index
+      b = buf[i].ord
+      i += 1
+      if (b <= 0x1f) || (b >= 0x7f)
+        out.printf(".")
+      else
+        out.printf("%c", b)
+      end
+    end
+    out.printf("|")
+    row += 1
+  end
+  if remainder != 0
+    row_start_index = full_rows * width
+    # prefix
+    out.printf("\n|%08x|", row_start_index)
+    row_end_index = row_start_index + remainder
+    # hex
+    i = row_start_index
+    while i < row_end_index
+      out.printf(" %02x", buf[i].ord)
+      i += 1
+    end
+    i = width - remainder
+    while i > 0
+      out.printf("   ")
+      i -= 1
+    end
+    out.printf(" |")
+    # ascii
+    i = row_start_index
+    while i < row_end_index
+      b = buf[i].ord
+      i += 1
+      if (b <= 0x1f) || (b >= 0x7f)
+        out.printf(".")
+      else
+        out.printf("%c", b)
+      end
+    end
+    i = width - remainder
+    while i > 0
+      out.printf(" ")
+      i -= 1
+    end
+    out.printf("|")
+  end
+  if style == :wide
+    out.print("\n+--------+-------------------------------------------------------------------------------------------------+--------------------------------+\n")
+  else
+    out.print("\n+--------+-------------------------------------------------+----------------+\n")
+  end
+  out.string
+end
+
+ARGF.each_line do |line|
+  line.force_encoding(Encoding::BINARY)
+  md = line.match(/TRACE.*<([^>]+)>.*\(CTX=(0x[0-9a-z]+),([^,)]+).*pkt,(snd|rcv).*: size=(\d+), (.+)/)
+  next unless md
+  address = md[1]
+  ctx_id = md[2]
+  subsys = md[3]
+  dir = md[4]
+  data = Base64.decode64(md[6])
+  puts pad(dir, "#{address} CTX=#{ctx_id},#{subsys} #{dir == 'snd' ? 'sent' : 'received'} #{data.size} bytes")
+  puts pad(dir, format_bytes(data))
+end