couchbase 3.0.0.alpha.1 → 3.0.0.alpha.2

data/ext/third_party/snappy/cmake/config.h.in

@@ -0,0 +1,59 @@
+#ifndef THIRD_PARTY_SNAPPY_OPENSOURCE_CMAKE_CONFIG_H_
+#define THIRD_PARTY_SNAPPY_OPENSOURCE_CMAKE_CONFIG_H_
+
+/* Define to 1 if the compiler supports __attribute__((always_inline)). */
+#cmakedefine HAVE_ATTRIBUTE_ALWAYS_INLINE 1
+
+/* Define to 1 if the compiler supports __builtin_ctz and friends. */
+#cmakedefine HAVE_BUILTIN_CTZ 1
+
+/* Define to 1 if the compiler supports __builtin_expect. */
+#cmakedefine HAVE_BUILTIN_EXPECT 1
+
+/* Define to 1 if you have a definition for mmap() in <sys/mman.h>. */
+#cmakedefine HAVE_FUNC_MMAP 1
+
+/* Define to 1 if you have a definition for sysconf() in <unistd.h>. */
+#cmakedefine HAVE_FUNC_SYSCONF 1
+
+/* Define to 1 to use the gflags package for command-line parsing. */
+#cmakedefine HAVE_GFLAGS 1
+
+/* Define to 1 if you have Google Test. */
+#cmakedefine HAVE_GTEST 1
+
+/* Define to 1 if you have the `lzo2' library (-llzo2). */
+#cmakedefine HAVE_LIBLZO2 1
+
+/* Define to 1 if you have the `z' library (-lz). */
+#cmakedefine HAVE_LIBZ 1
+
+/* Define to 1 if you have the <sys/mman.h> header file. */
+#cmakedefine HAVE_SYS_MMAN_H 1
+
+/* Define to 1 if you have the <sys/resource.h> header file. */
+#cmakedefine HAVE_SYS_RESOURCE_H 1
+
+/* Define to 1 if you have the <sys/time.h> header file. */
+#cmakedefine HAVE_SYS_TIME_H 1
+
+/* Define to 1 if you have the <sys/uio.h> header file. */
+#cmakedefine HAVE_SYS_UIO_H 1
+
+/* Define to 1 if you have the <unistd.h> header file. */
+#cmakedefine HAVE_UNISTD_H 1
+
+/* Define to 1 if you have the <windows.h> header file. */
+#cmakedefine HAVE_WINDOWS_H 1
+
+/* Define to 1 if you target processors with SSSE3+ and have <tmmintrin.h>. */
+#cmakedefine01 SNAPPY_HAVE_SSSE3
+
+/* Define to 1 if you target processors with BMI2+ and have <bmi2intrin.h>. */
+#cmakedefine01 SNAPPY_HAVE_BMI2
+
+/* Define to 1 if your processor stores words with the most significant byte
+   first (like Motorola and SPARC, unlike Intel and VAX). */
+#cmakedefine SNAPPY_IS_BIG_ENDIAN 1
+
+#endif  // THIRD_PARTY_SNAPPY_OPENSOURCE_CMAKE_CONFIG_H_

data/ext/third_party/snappy/docs/README.md

@@ -0,0 +1,72 @@
+Snappy is a compression/decompression library. It does not aim for maximum
+compression, or compatibility with any other compression library; instead, it
+aims for very high speeds and reasonable compression. For instance, compared
+to the fastest mode of zlib, Snappy is an order of magnitude faster for most
+inputs, but the resulting compressed files are anywhere from 20% to 100%
+bigger. On a single core of a Core i7 processor in 64-bit mode, Snappy
+compresses at about 250 MB/sec or more and decompresses at about 500 MB/sec
+or more.
+
+Snappy is widely used inside Google, in everything from BigTable and MapReduce
+to our internal RPC systems. (Snappy has previously been referred to as "Zippy"
+in some presentations and the likes.)
+
+For more information, please see the [README](../README.md). Benchmarks against
+a few other compression libraries (zlib, LZO, LZF, FastLZ, and QuickLZ) are
+included in the source code distribution. The source code also contains a
+[formal format specification](../format_description.txt), as well
+as a specification for a [framing format](../framing_format.txt) useful for
+higher-level framing and encapsulation of Snappy data, e.g. for transporting
+Snappy-compressed data across HTTP in a streaming fashion. Note that the Snappy
+distribution currently has no code implementing the latter, but some of the
+ports do (see below).
+
+Snappy is written in C++, but C bindings are included, and several bindings to
+other languages are maintained by third parties:
+
+* C#: [Snappy for .NET](http://snappy4net.codeplex.com/) (P/Invoke wrapper),
+  [Snappy.NET](http://snappy.angeloflogic.com/) (P/Invoke wrapper),
+  [Snappy.Sharp](https://github.com/jeffesp/Snappy.Sharp) (native
+  reimplementation)
+* [C port](http://github.com/andikleen/snappy-c)
+* [C++ MSVC packaging](http://snappy.angeloflogic.com/) (plus Windows binaries,
+  NuGet packages and command-line tool)
+* Common Lisp: [Library bindings](http://flambard.github.com/thnappy/),
+  [native reimplementation](https://github.com/brown/snappy)
+* Erlang: [esnappy](https://github.com/thekvs/esnappy),
+  [snappy-erlang-nif](https://github.com/fdmanana/snappy-erlang-nif)
+* [Go](https://github.com/golang/snappy/)
+* [Haskell](http://hackage.haskell.org/package/snappy)
+* [Haxe](https://github.com/MaddinXx/hxsnappy) (C++/Neko)
+* [iOS packaging](https://github.com/ideawu/snappy-ios)
+* Java: [JNI wrapper](https://github.com/xerial/snappy-java) (including the
+  framing format), [native reimplementation](http://code.google.com/p/jsnappy/),
+  [other native reimplementation](https://github.com/dain/snappy) (including
+  the framing format)
+* [Lua](https://github.com/forhappy/lua-snappy)
+* [Node.js](https://github.com/kesla/node-snappy) (including the [framing
+  format](https://github.com/kesla/node-snappy-stream))
+* [Perl](http://search.cpan.org/dist/Compress-Snappy/)
+* [PHP](https://github.com/kjdev/php-ext-snappy)
+* [Python](http://pypi.python.org/pypi/python-snappy) (including a command-line
+  tool for the framing format)
+* [R](https://github.com/lulyon/R-snappy)
+* [Ruby](https://github.com/miyucy/snappy)
+* [Rust](https://github.com/BurntSushi/rust-snappy)
+* [Smalltalk](https://github.com/mumez/sqnappy) (including the framing format)
+
+Snappy is used or is available as an alternative in software such as
+
+* [MongoDB](https://www.mongodb.com/)
+* [Cassandra](http://cassandra.apache.org/)
+* [Couchbase](http://www.couchbase.com/)
+* [Hadoop](http://hadoop.apache.org/)
+* [LessFS](http://www.lessfs.com/wordpress/)
+* [LevelDB](https://github.com/google/leveldb) (which is in turn used by
+  [Google Chrome](http://chrome.google.com/))
+* [Lucene](http://lucene.apache.org/)
+* [VoltDB](http://voltdb.com/)
+
+If you know of more, do not hesitate to let us know. The easiest way to get in
+touch is via the
+[Snappy discussion mailing list](http://groups.google.com/group/snappy-compression).

data/ext/third_party/snappy/format_description.txt

@@ -0,0 +1,110 @@
+Snappy compressed format description
+Last revised: 2011-10-05
+
+
+This is not a formal specification, but should suffice to explain most
+relevant parts of how the Snappy format works. It is originally based on
+text by Zeev Tarantov.
+
+Snappy is a LZ77-type compressor with a fixed, byte-oriented encoding.
+There is no entropy encoder backend nor framing layer -- the latter is
+assumed to be handled by other parts of the system.
+
+This document only describes the format, not how the Snappy compressor nor
+decompressor actually works. The correctness of the decompressor should not
+depend on implementation details of the compressor, and vice versa.
+
+
+1. Preamble
+
+The stream starts with the uncompressed length (up to a maximum of 2^32 - 1),
+stored as a little-endian varint. Varints consist of a series of bytes,
+where the lower 7 bits are data and the upper bit is set iff there are
+more bytes to be read. In other words, an uncompressed length of 64 would
+be stored as 0x40, and an uncompressed length of 2097150 (0x1FFFFE)
+would be stored as 0xFE 0xFF 0x7F.
+
+
+2. The compressed stream itself
+
+There are two types of elements in a Snappy stream: Literals and
+copies (backreferences). There is no restriction on the order of elements,
+except that the stream naturally cannot start with a copy. (Having
+two literals in a row is never optimal from a compression point of
+view, but nevertheless fully permitted.) Each element starts with a tag byte,
+and the lower two bits of this tag byte signal what type of element will
+follow:
+
+  00: Literal
+  01: Copy with 1-byte offset
+  10: Copy with 2-byte offset
+  11: Copy with 4-byte offset
+
+The interpretation of the upper six bits are element-dependent.
+
+
+2.1. Literals (00)
+
+Literals are uncompressed data stored directly in the byte stream.
+The literal length is stored differently depending on the length
+of the literal:
+
+ - For literals up to and including 60 bytes in length, the upper
+   six bits of the tag byte contain (len-1). The literal follows
+   immediately thereafter in the bytestream.
+ - For longer literals, the (len-1) value is stored after the tag byte,
+   little-endian. The upper six bits of the tag byte describe how
+   many bytes are used for the length; 60, 61, 62 or 63 for
+   1-4 bytes, respectively. The literal itself follows after the
+   length.
+
+
+2.2. Copies
+
+Copies are references back into previous decompressed data, telling
+the decompressor to reuse data it has previously decoded.
+They encode two values: The _offset_, saying how many bytes back
+from the current position to read, and the _length_, how many bytes
+to copy. Offsets of zero can be encoded, but are not legal;
+similarly, it is possible to encode backreferences that would
+go past the end of the block (offset > current decompressed position),
+which is also nonsensical and thus not allowed.
+
+As in most LZ77-based compressors, the length can be larger than the offset,
+yielding a form of run-length encoding (RLE). For instance,
+"xababab" could be encoded as
+
+  <literal: "xab"> <copy: offset=2 length=4>
+
+Note that since the current Snappy compressor works in 32 kB
+blocks and does not do matching across blocks, it will never produce
+a bitstream with offsets larger than about 32768. However, the
+decompressor should not rely on this, as it may change in the future.
+
+There are several different kinds of copy elements, depending on
+the amount of bytes to be copied (length), and how far back the
+data to be copied is (offset).
+
+
+2.2.1. Copy with 1-byte offset (01)
+
+These elements can encode lengths between [4..11] bytes and offsets
+between [0..2047] bytes. (len-4) occupies three bits and is stored
+in bits [2..4] of the tag byte. The offset occupies 11 bits, of which the
+upper three are stored in the upper three bits ([5..7]) of the tag byte,
+and the lower eight are stored in a byte following the tag byte.
+
+
+2.2.2. Copy with 2-byte offset (10)
+
+These elements can encode lengths between [1..64] and offsets from
+[0..65535]. (len-1) occupies six bits and is stored in the upper
+six bits ([2..7]) of the tag byte. The offset is stored as a
+little-endian 16-bit integer in the two bytes following the tag byte.
+
+
+2.2.3. Copy with 4-byte offset (11)
+
+These are like the copies with 2-byte offsets (see previous subsection),
+except that the offset is stored as a 32-bit integer instead of a
+16-bit integer (and thus will occupy four bytes).

data/ext/third_party/snappy/framing_format.txt

@@ -0,0 +1,135 @@
+Snappy framing format description
+Last revised: 2013-10-25
+
+This format decribes a framing format for Snappy, allowing compressing to
+files or streams that can then more easily be decompressed without having
+to hold the entire stream in memory. It also provides data checksums to
+help verify integrity. It does not provide metadata checksums, so it does
+not protect against e.g. all forms of truncations.
+
+Implementation of the framing format is optional for Snappy compressors and
+decompressor; it is not part of the Snappy core specification.
+
+
+1. General structure
+
+The file consists solely of chunks, lying back-to-back with no padding
+in between. Each chunk consists first a single byte of chunk identifier,
+then a three-byte little-endian length of the chunk in bytes (from 0 to
+16777215, inclusive), and then the data if any. The four bytes of chunk
+header is not counted in the data length.
+
+The different chunk types are listed below. The first chunk must always
+be the stream identifier chunk (see section 4.1, below). The stream
+ends when the file ends -- there is no explicit end-of-file marker.
+
+
+2. File type identification
+
+The following identifiers for this format are recommended where appropriate.
+However, note that none have been registered officially, so this is only to
+be taken as a guideline. We use "Snappy framed" to distinguish between this
+format and raw Snappy data.
+
+  File extension:         .sz
+  MIME type:              application/x-snappy-framed
+  HTTP Content-Encoding:  x-snappy-framed
+
+
+3. Checksum format
+
+Some chunks have data protected by a checksum (the ones that do will say so
+explicitly). The checksums are always masked CRC-32Cs.
+
+A description of CRC-32C can be found in RFC 3720, section 12.1, with
+examples in section B.4.
+
+Checksums are not stored directly, but masked, as checksumming data and
+then its own checksum can be problematic. The masking is the same as used
+in Apache Hadoop: Rotate the checksum by 15 bits, then add the constant
+0xa282ead8 (using wraparound as normal for unsigned integers). This is
+equivalent to the following C code:
+
+  uint32_t mask_checksum(uint32_t x) {
+    return ((x >> 15) | (x << 17)) + 0xa282ead8;
+  }
+
+Note that the masking is reversible.
+
+The checksum is always stored as a four bytes long integer, in little-endian.
+
+
+4. Chunk types
+
+The currently supported chunk types are described below. The list may
+be extended in the future.
+
+
+4.1. Stream identifier (chunk type 0xff)
+
+The stream identifier is always the first element in the stream.
+It is exactly six bytes long and contains "sNaPpY" in ASCII. This means that
+a valid Snappy framed stream always starts with the bytes
+
+  0xff 0x06 0x00 0x00 0x73 0x4e 0x61 0x50 0x70 0x59
+
+The stream identifier chunk can come multiple times in the stream besides
+the first; if such a chunk shows up, it should simply be ignored, assuming
+it has the right length and contents. This allows for easy concatenation of
+compressed files without the need for re-framing.
+
+
+4.2. Compressed data (chunk type 0x00)
+
+Compressed data chunks contain a normal Snappy compressed bitstream;
+see the compressed format specification. The compressed data is preceded by
+the CRC-32C (see section 3) of the _uncompressed_ data.
+
+Note that the data portion of the chunk, i.e., the compressed contents,
+can be at most 16777211 bytes (2^24 - 1, minus the checksum).
+However, we place an additional restriction that the uncompressed data
+in a chunk must be no longer than 65536 bytes. This allows consumers to
+easily use small fixed-size buffers.
+
+
+4.3. Uncompressed data (chunk type 0x01)
+
+Uncompressed data chunks allow a compressor to send uncompressed,
+raw data; this is useful if, for instance, uncompressible or
+near-incompressible data is detected, and faster decompression is desired.
+
+As in the compressed chunks, the data is preceded by its own masked
+CRC-32C (see section 3).
+
+An uncompressed data chunk, like compressed data chunks, should contain
+no more than 65536 data bytes, so the maximum legal chunk length with the
+checksum is 65540.
+
+
+4.4. Padding (chunk type 0xfe)
+
+Padding chunks allow a compressor to increase the size of the data stream
+so that it complies with external demands, e.g. that the total number of
+bytes is a multiple of some value.
+
+All bytes of the padding chunk, except the chunk byte itself and the length,
+should be zero, but decompressors must not try to interpret or verify the
+padding data in any way.
+
+
+4.5. Reserved unskippable chunks (chunk types 0x02-0x7f)
+
+These are reserved for future expansion. A decoder that sees such a chunk
+should immediately return an error, as it must assume it cannot decode the
+stream correctly.
+
+Future versions of this specification may define meanings for these chunks.
+
+
+4.6. Reserved skippable chunks (chunk types 0x80-0xfd)
+
+These are also reserved for future expansion, but unlike the chunks
+described in 4.5, a decoder seeing these must skip them and continue
+decoding.
+
+Future versions of this specification may define meanings for these chunks.

data/ext/third_party/snappy/snappy-c.cc

@@ -0,0 +1,90 @@
+// Copyright 2011 Martin Gieseking <martin.gieseking@uos.de>.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "snappy.h"
+#include "snappy-c.h"
+
+extern "C" {
+
+snappy_status snappy_compress(const char* input,
+                              size_t input_length,
+                              char* compressed,
+                              size_t *compressed_length) {
+  if (*compressed_length < snappy_max_compressed_length(input_length)) {
+    return SNAPPY_BUFFER_TOO_SMALL;
+  }
+  snappy::RawCompress(input, input_length, compressed, compressed_length);
+  return SNAPPY_OK;
+}
+
+snappy_status snappy_uncompress(const char* compressed,
+                                size_t compressed_length,
+                                char* uncompressed,
+                                size_t* uncompressed_length) {
+  size_t real_uncompressed_length;
+  if (!snappy::GetUncompressedLength(compressed,
+                                     compressed_length,
+                                     &real_uncompressed_length)) {
+    return SNAPPY_INVALID_INPUT;
+  }
+  if (*uncompressed_length < real_uncompressed_length) {
+    return SNAPPY_BUFFER_TOO_SMALL;
+  }
+  if (!snappy::RawUncompress(compressed, compressed_length, uncompressed)) {
+    return SNAPPY_INVALID_INPUT;
+  }
+  *uncompressed_length = real_uncompressed_length;
+  return SNAPPY_OK;
+}
+
+size_t snappy_max_compressed_length(size_t source_length) {
+  return snappy::MaxCompressedLength(source_length);
+}
+
+snappy_status snappy_uncompressed_length(const char *compressed,
+                                         size_t compressed_length,
+                                         size_t *result) {
+  if (snappy::GetUncompressedLength(compressed,
+                                    compressed_length,
+                                    result)) {
+    return SNAPPY_OK;
+  } else {
+    return SNAPPY_INVALID_INPUT;
+  }
+}
+
+snappy_status snappy_validate_compressed_buffer(const char *compressed,
+                                                size_t compressed_length) {
+  if (snappy::IsValidCompressedBuffer(compressed, compressed_length)) {
+    return SNAPPY_OK;
+  } else {
+    return SNAPPY_INVALID_INPUT;
+  }
+}
+
+}  // extern "C"