snappy 0.0.17 → 0.1.0

data/test/hadoop/test-snappy-hadoop-writer.rb

@@ -0,0 +1,48 @@
+require "minitest/autorun"
+require "minitest/spec"
+require "snappy"
+require "stringio"
+
+describe Snappy::Hadoop::Writer do
+  before do
+    @buffer = StringIO.new
+  end
+
+  subject do
+    Snappy::Hadoop::Writer.new @buffer
+  end
+
+  describe :initialize do
+    it "should yield itself to the block" do
+      yielded = nil
+      returned = Snappy::Hadoop::Writer.new @buffer do |w|
+        yielded = w
+      end
+      returned.must_equal yielded
+    end
+
+    it "should dump on the end of yield" do
+      Snappy::Hadoop::Writer.new @buffer do |w|
+        w << "foo"
+      end
+      @buffer.string.must_equal "\u0000\u0000\u0000\u0003\u0000\u0000\u0000\u0005\u0003\bfoo"
+    end
+  end
+
+  describe :io do
+    it "should be a constructor argument" do
+      io = StringIO.new
+      Snappy::Hadoop::Writer.new(io).io.must_equal io
+    end
+  end
+
+  describe :block_size do
+    it "should default to DEFAULT_BLOCK_SIZE" do
+      Snappy::Hadoop::Writer.new(StringIO.new).block_size.must_equal Snappy::Hadoop::Writer::DEFAULT_BLOCK_SIZE
+    end
+
+    it "should be settable via the constructor" do
+      Snappy::Hadoop::Writer.new(StringIO.new, 42).block_size.must_equal 42
+    end
+  end
+end

data/test/test-snappy-hadoop.rb

@@ -0,0 +1,22 @@
+require 'minitest/autorun'
+require 'minitest/spec'
+require 'snappy'
+
+describe Snappy::Hadoop do
+  T = [*'a'..'z', *'A'..'Z', *'0'..'9']
+
+  it 'well done' do
+    s = Array.new(1024){T.sample}.join
+    Snappy::Hadoop.inflate(Snappy::Hadoop.deflate s).must_equal(s)
+  end
+
+  it 'well done (pair)' do
+    s = Array.new(1024){T.sample}.join
+    [
+     [:deflate,  :inflate],
+    ].each do |(i, o)|
+      Snappy::Hadoop.__send__(o, (Snappy::Hadoop.__send__ i,  s)).must_equal(s)
+      eval %{Snappy::Hadoop.#{o}(Snappy::Hadoop.#{i} s).must_equal(s)}
+    end
+  end
+end

data/vendor/snappy/CMakeLists.txt

@@ -0,0 +1,174 @@
+cmake_minimum_required(VERSION 3.1)
+project(Snappy VERSION 1.1.7 LANGUAGES C CXX)
+
+# BUILD_SHARED_LIBS is a standard CMake variable, but we declare it here to make
+# it prominent in the GUI.
+option(BUILD_SHARED_LIBS "Build shared libraries(DLLs)." OFF)
+
+option(SNAPPY_BUILD_TESTS "Build Snappy's own tests." ON)
+
+include(TestBigEndian)
+test_big_endian(SNAPPY_IS_BIG_ENDIAN)
+
+include(CheckIncludeFile)
+check_include_file("byteswap.h" HAVE_BYTESWAP_H)
+check_include_file("stddef.h" HAVE_STDDEF_H)
+check_include_file("stdint.h" HAVE_STDINT_H)
+check_include_file("sys/endian.h" HAVE_SYS_ENDIAN_H)
+check_include_file("sys/mman.h" HAVE_SYS_MMAN_H)
+check_include_file("sys/resource.h" HAVE_SYS_RESOURCE_H)
+check_include_file("sys/time.h" HAVE_SYS_TIME_H)
+check_include_file("sys/uio.h" HAVE_SYS_UIO_H)
+check_include_file("unistd.h" HAVE_UNISTD_H)
+check_include_file("windows.h" HAVE_WINDOWS_H)
+
+include(CheckLibraryExists)
+check_library_exists(z zlibVersion "" HAVE_LIBZ)
+check_library_exists(lzo2 lzo1x_1_15_compress "" HAVE_LIBLZO2)
+
+include(CheckCXXSourceCompiles)
+check_cxx_source_compiles(
+    "int main(void) { return __builtin_expect(0, 1); }" HAVE_BUILTIN_EXPECT)
+
+check_cxx_source_compiles(
+    "int main(void) { return __builtin_ctzll(0); }" HAVE_BUILTIN_CTZ)
+
+include(CheckSymbolExists)
+check_symbol_exists("mmap" "sys/mman.h" HAVE_FUNC_MMAP)
+check_symbol_exists("sysconf" "unistd.h" HAVE_FUNC_SYSCONF)
+
+find_package(GTest QUIET)
+if(GTEST_FOUND)
+  set(HAVE_GTEST 1)
+endif(GTEST_FOUND)
+
+find_package(Gflags QUIET)
+if(GFLAGS_FOUND)
+  set(HAVE_GFLAGS 1)
+endif(GFLAGS_FOUND)
+
+configure_file(
+  "${PROJECT_SOURCE_DIR}/cmake/config.h.in"
+  "${PROJECT_BINARY_DIR}/config.h"
+)
+
+# We don't want to define HAVE_ macros in public headers. Instead, we use
+# CMake's variable substitution with 0/1 variables, which will be seen by the
+# preprocessor as constants.
+set(HAVE_STDINT_H_01  ${HAVE_STDINT_H})
+set(HAVE_STDDEF_H_01  ${HAVE_STDDEF_H})
+set(HAVE_SYS_UIO_H_01 ${HAVE_SYS_UIO_H})
+if(NOT HAVE_STDINT_H_01)
+  set(HAVE_STDINT_H_01 0)
+endif(NOT HAVE_STDINT_H_01)
+if(NOT HAVE_STDDEF_H_01)
+  set(HAVE_STDDEF_H_01 0)
+endif(NOT HAVE_STDDEF_H_01)
+if(NOT HAVE_SYS_UIO_H_01)
+  set(HAVE_SYS_UIO_H_01 0)
+endif(NOT HAVE_SYS_UIO_H_01)
+
+configure_file(
+  "${PROJECT_SOURCE_DIR}/snappy-stubs-public.h.in"
+  "${PROJECT_BINARY_DIR}/snappy-stubs-public.h")
+
+add_library(snappy "")
+target_sources(snappy
+  PRIVATE
+    "${PROJECT_SOURCE_DIR}/snappy-internal.h"
+    "${PROJECT_SOURCE_DIR}/snappy-stubs-internal.h"
+    "${PROJECT_SOURCE_DIR}/snappy-c.cc"
+    "${PROJECT_SOURCE_DIR}/snappy-sinksource.cc"
+    "${PROJECT_SOURCE_DIR}/snappy-stubs-internal.cc"
+    "${PROJECT_SOURCE_DIR}/snappy.cc"
+    "${PROJECT_BINARY_DIR}/config.h"
+
+  # Only CMake 3.3+ supports PUBLIC sources in targets exported by "install".
+  $<$<VERSION_GREATER:CMAKE_VERSION,3.2>:PUBLIC>
+    $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/snappy-c.h>
+    $<INSTALL_INTERFACE:include/snappy-c.h>
+    $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/snappy-sinksource.h>
+    $<INSTALL_INTERFACE:include/snappy-sinksource.h>
+    $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/snappy.h>
+    $<INSTALL_INTERFACE:include/snappy.h>
+    $<BUILD_INTERFACE:${PROJECT_BINARY_DIR}/snappy-stubs-public.h>
+    $<INSTALL_INTERFACE:include/snappy-stubs-public.h>
+)
+target_include_directories(snappy
+  PUBLIC
+    $<BUILD_INTERFACE:${PROJECT_BINARY_DIR}>
+    $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}>
+    $<INSTALL_INTERFACE:include>
+)
+set_target_properties(snappy
+  PROPERTIES VERSION ${PROJECT_VERSION} SOVERSION ${PROJECT_VERSION_MAJOR})
+
+target_compile_definitions(snappy PRIVATE -DHAVE_CONFIG_H)
+if(BUILD_SHARED_LIBS)
+  set_target_properties(snappy PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON)
+endif(BUILD_SHARED_LIBS)
+
+if(SNAPPY_BUILD_TESTS)
+  enable_testing()
+
+  add_executable(snappy_unittest "")
+  target_sources(snappy_unittest
+    PRIVATE
+      "${PROJECT_SOURCE_DIR}/snappy_unittest.cc"
+      "${PROJECT_SOURCE_DIR}/snappy-test.cc"
+  )
+  target_compile_definitions(snappy_unittest PRIVATE -DHAVE_CONFIG_H)
+  target_link_libraries(snappy_unittest snappy ${GFLAGS_LIBRARIES})
+
+  if(HAVE_LIBZ)
+    target_link_libraries(snappy_unittest z)
+  endif(HAVE_LIBZ)
+  if(HAVE_LIBLZO2)
+    target_link_libraries(snappy_unittest lzo2)
+  endif(HAVE_LIBLZO2)
+
+  target_include_directories(snappy_unittest
+    BEFORE PRIVATE
+      "${PROJECT_SOURCE_DIR}"
+      "${GTEST_INCLUDE_DIRS}"
+      "${GFLAGS_INCLUDE_DIRS}"
+  )
+
+  add_test(
+    NAME snappy_unittest
+    WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}"
+    COMMAND "${PROJECT_BINARY_DIR}/snappy_unittest")
+endif(SNAPPY_BUILD_TESTS)
+
+include(GNUInstallDirs)
+install(TARGETS snappy
+  EXPORT SnappyTargets
+  RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+  LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+  ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+)
+install(
+  FILES
+    "${PROJECT_SOURCE_DIR}/snappy-c.h"
+    "${PROJECT_SOURCE_DIR}/snappy-sinksource.h"
+    "${PROJECT_SOURCE_DIR}/snappy.h"
+    "${PROJECT_BINARY_DIR}/snappy-stubs-public.h"
+  DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
+)
+
+include(CMakePackageConfigHelpers)
+write_basic_package_version_file(
+    "${PROJECT_BINARY_DIR}/SnappyConfigVersion.cmake"
+    COMPATIBILITY SameMajorVersion
+)
+install(
+  EXPORT SnappyTargets
+  NAMESPACE Snappy::
+  DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/Snappy"
+)
+install(
+  FILES
+    "${PROJECT_SOURCE_DIR}/cmake/SnappyConfig.cmake"
+    "${PROJECT_BINARY_DIR}/SnappyConfigVersion.cmake"
+  DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/Snappy"
+)

data/vendor/snappy/CONTRIBUTING.md

@@ -0,0 +1,26 @@
+# How to Contribute
+
+We'd love to accept your patches and contributions to this project. There are
+just a few small guidelines you need to follow.
+
+## Contributor License Agreement
+
+Contributions to this project must be accompanied by a Contributor License
+Agreement. You (or your employer) retain the copyright to your contribution,
+this simply gives us permission to use and redistribute your contributions as
+part of the project. Head over to <https://cla.developers.google.com/> to see
+your current agreements on file or to sign a new one.
+
+You generally only need to submit a CLA once, so if you've already submitted one
+(even if it was for a different project), you probably don't need to do it
+again.
+
+## Code reviews
+
+All submissions, including submissions by project members, require review. We
+use GitHub pull requests for this purpose. Consult
+[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more
+information on using pull requests.
+
+Please make sure that all the automated checks (CLA, AppVeyor, Travis) pass for
+your pull requests. Pull requests whose checks fail may be ignored.

data/vendor/snappy/NEWS

@@ -1,3 +1,35 @@
+Snappy v1.1.7, August 24th 2017:
+
+  * Improved CMake build support for 64-bit Linux distributions.
+
+  * MSVC builds now use MSVC-specific intrinsics that map to clzll.
+
+  * ARM64 (AArch64) builds use the code paths optimized for 64-bit processors.
+
+Snappy v1.1.6, July 12th 2017:
+
+This is a re-release of v1.1.5 with proper SONAME / SOVERSION values.
+
+Snappy v1.1.5, June 28th 2017:
+
+This release has broken SONAME / SOVERSION values. Users of snappy as a shared
+library should avoid 1.1.5 and use 1.1.6 instead. SONAME / SOVERSION errors will
+manifest as the dynamic library loader complaining that it cannot find snappy's
+shared library file (libsnappy.so / libsnappy.dylib), or that the library it
+found does not have the required version. 1.1.6 has the same code as 1.1.5, but
+carries build configuration fixes for the issues above.
+
+  * Add CMake build support. The autoconf build support is now deprecated, and
+    will be removed in the next release.
+
+  * Add AppVeyor configuration, for Windows CI coverage.
+
+  * Small performance improvement on little-endian PowerPC.
+
+  * Small performance improvement on LLVM with position-independent executables.
+
+  * Fix a few issues with various build environments.
+
 Snappy v1.1.4, January 25th 2017:
 
   * Fix a 1% performance regression when snappy is used in PIE executables.

data/vendor/snappy/{README → README.md}

@@ -34,7 +34,7 @@ Snappy is intended to be fast. On a single core of a Core i7 processor
 in 64-bit mode, it compresses at about 250 MB/sec or more and decompresses at
 about 500 MB/sec or more. (These numbers are for the slowest inputs in our
 benchmark suite; others are much faster.) In our tests, Snappy usually
-is faster than algorithms in the same class (e.g. LZO, LZF, FastLZ, QuickLZ,
+is faster than algorithms in the same class (e.g. LZO, LZF, QuickLZ,
 etc.) while achieving comparable compression ratios.
 
 Typical compression ratios (based on the benchmark suite) are about 1.5-1.7x
@@ -52,7 +52,7 @@ In particular:
  - Snappy uses 64-bit operations in several places to process more data at
    once than would otherwise be possible.
  - Snappy assumes unaligned 32- and 64-bit loads and stores are cheap.
-   On some platforms, these must be emulated with single-byte loads 
+   On some platforms, these must be emulated with single-byte loads
    and stores, which is much slower.
  - Snappy assumes little-endian throughout, and needs to byte-swap data in
    several places if running on a big-endian platform.
@@ -62,6 +62,16 @@ Performance optimizations, whether for 64-bit x86 or other platforms,
 are of course most welcome; see "Contact", below.
 
 
+Building
+========
+
+CMake is supported and autotools will soon be deprecated.
+You need CMake 3.4 or above to build:
+
+  mkdir build
+  cd build && cmake ../ && make
+
+
 Usage
 =====
 
@@ -116,7 +126,7 @@ before the unit tests, but you can disable them using the flag
 need to edit the source).
 
 Finally, snappy can benchmark Snappy against a few other compression libraries
-(zlib, LZO, LZF, FastLZ and QuickLZ), if they were detected at configure time.
+(zlib, LZO, LZF, and QuickLZ), if they were detected at configure time.
 To benchmark using a given file, give the compression algorithm you want to test
 Snappy against (e.g. --zlib) and then a list of one or more file names on the
 command line. The testdata/ directory contains the files used by the

data/vendor/snappy/cmake/SnappyConfig.cmake

@@ -0,0 +1 @@
+include("${CMAKE_CURRENT_LIST_DIR}/SnappyTargets.cmake")

data/vendor/snappy/cmake/config.h.in

@@ -0,0 +1,62 @@
+#ifndef THIRD_PARTY_SNAPPY_OPENSOURCE_CMAKE_CONFIG_H_
+#define THIRD_PARTY_SNAPPY_OPENSOURCE_CMAKE_CONFIG_H_
+
+/* Define to 1 if the compiler supports __builtin_ctz and friends. */
+#cmakedefine HAVE_BUILTIN_CTZ 1
+
+/* Define to 1 if the compiler supports __builtin_expect. */
+#cmakedefine HAVE_BUILTIN_EXPECT 1
+
+/* Define to 1 if you have the <byteswap.h> header file. */
+#cmakedefine HAVE_BYTESWAP_H 1
+
+/* Define to 1 if you have a definition for mmap() in <sys/mman.h>. */
+#cmakedefine HAVE_FUNC_MMAP 1
+
+/* Define to 1 if you have a definition for sysconf() in <unistd.h>. */
+#cmakedefine HAVE_FUNC_SYSCONF 1
+
+/* Define to 1 to use the gflags package for command-line parsing. */
+#cmakedefine HAVE_GFLAGS 1
+
+/* Define to 1 if you have Google Test. */
+#cmakedefine HAVE_GTEST 1
+
+/* Define to 1 if you have the `lzo2' library (-llzo2). */
+#cmakedefine HAVE_LIBLZO2 1
+
+/* Define to 1 if you have the `z' library (-lz). */
+#cmakedefine HAVE_LIBZ 1
+
+/* Define to 1 if you have the <stddef.h> header file. */
+#cmakedefine HAVE_STDDEF_H 1
+
+/* Define to 1 if you have the <stdint.h> header file. */
+#cmakedefine HAVE_STDINT_H 1
+
+/* Define to 1 if you have the <sys/endian.h> header file. */
+#cmakedefine HAVE_SYS_ENDIAN_H 1
+
+/* Define to 1 if you have the <sys/mman.h> header file. */
+#cmakedefine HAVE_SYS_MMAN_H 1
+
+/* Define to 1 if you have the <sys/resource.h> header file. */
+#cmakedefine HAVE_SYS_RESOURCE_H 1
+
+/* Define to 1 if you have the <sys/time.h> header file. */
+#cmakedefine HAVE_SYS_TIME_H 1
+
+/* Define to 1 if you have the <sys/uio.h> header file. */
+#cmakedefine HAVE_SYS_UIO_H 1
+
+/* Define to 1 if you have the <unistd.h> header file. */
+#cmakedefine HAVE_UNISTD_H 1
+
+/* Define to 1 if you have the <windows.h> header file. */
+#cmakedefine HAVE_WINDOWS_H 1
+
+/* Define to 1 if your processor stores words with the most significant byte
+   first (like Motorola and SPARC, unlike Intel and VAX). */
+#cmakedefine SNAPPY_IS_BIG_ENDIAN 1
+
+#endif  // THIRD_PARTY_SNAPPY_OPENSOURCE_CMAKE_CONFIG_H_

data/vendor/snappy/snappy-internal.h

@@ -50,7 +50,9 @@ class WorkingMemory {
   uint16 small_table_[1<<10];    // 2KB
   uint16* large_table_;          // Allocated only when needed
 
-  DISALLOW_COPY_AND_ASSIGN(WorkingMemory);
+  // No copying
+  WorkingMemory(const WorkingMemory&);
+  void operator=(const WorkingMemory&);
 };
 
 // Flat array compression that does not emit the "uncompressed length"
@@ -80,9 +82,9 @@ char* CompressFragment(const char* input,
 // Does not read *(s1 + (s2_limit - s2)) or beyond.
 // Requires that s2_limit >= s2.
 //
-// Separate implementation for x86_64, for speed.  Uses the fact that
-// x86_64 is little endian.
-#if defined(ARCH_K8)
+// Separate implementation for 64-bit, little-endian cpus.
+#if !defined(SNAPPY_IS_BIG_ENDIAN) && \
+    (defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM))
 static inline std::pair<size_t, bool> FindMatchLength(const char* s1,
                                                       const char* s2,
                                                       const char* s2_limit) {
@@ -94,7 +96,7 @@ static inline std::pair<size_t, bool> FindMatchLength(const char* s1,
   // uncommon code paths that determine, without extra effort, whether the match
   // length is less than 8.  In short, we are hoping to avoid a conditional
   // branch, and perhaps get better code layout from the C++ compiler.
-  if (PREDICT_TRUE(s2 <= s2_limit - 8)) {
+  if (SNAPPY_PREDICT_TRUE(s2 <= s2_limit - 8)) {
     uint64 a1 = UNALIGNED_LOAD64(s1);
     uint64 a2 = UNALIGNED_LOAD64(s2);
     if (a1 != a2) {
@@ -110,7 +112,7 @@ static inline std::pair<size_t, bool> FindMatchLength(const char* s1,
   // time until we find a 64-bit block that doesn't match; then we find
   // the first non-matching bit and use that to calculate the total
   // length of the match.
-  while (PREDICT_TRUE(s2 <= s2_limit - 8)) {
+  while (SNAPPY_PREDICT_TRUE(s2 <= s2_limit - 8)) {
     if (UNALIGNED_LOAD64(s2) == UNALIGNED_LOAD64(s1 + matched)) {
       s2 += 8;
       matched += 8;
@@ -122,7 +124,7 @@ static inline std::pair<size_t, bool> FindMatchLength(const char* s1,
       return std::pair<size_t, bool>(matched, false);
     }
   }
-  while (PREDICT_TRUE(s2 < s2_limit)) {
+  while (SNAPPY_PREDICT_TRUE(s2 < s2_limit)) {
     if (s1[matched] == *s2) {
       ++s2;
       ++matched;
@@ -170,11 +172,6 @@ enum {
 };
 static const int kMaximumTagLength = 5;  // COPY_4_BYTE_OFFSET plus the actual offset.
 
-// Mapping from i in range [0,4] to a mask to extract the bottom 8*i bits
-static const uint32 wordmask[] = {
-  0u, 0xffu, 0xffffu, 0xffffffu, 0xffffffffu
-};
-
 // Data stored per entry in lookup table:
 //      Range   Bits-used       Description
 //      ------------------------------------