extzstd 0.1 → 0.3.2

data/lib/extzstd/version.rb

@@ -1,3 +1,8 @@
 module Zstd
-  VERSION = "0.1"
+  verreg = /^\s*[\-\*]\s+version:\s*(\d+(?:\.\w+)+)\s*$/i
+  unless File.read("#{__dir__}/../../README.md", mode: "rt") =~ verreg
+    raise "``version'' is not defined or bad syntax in ``README.md''"
+  end
+
+  VERSION = String($1)
 end

data/test/test_basic.rb

@@ -9,11 +9,6 @@ class TestZstd < Test::Unit::TestCase
     src = "ABCDEFGabcdefg" * 50
     assert_equal(src, Zstd.decode(Zstd.encode(src), src.bytesize))
     #assert_raise(Zstd::Error) { Zstd.decode("", 1111) }
-    d1 = Zstd.encode(src)
-    assert_same(src.tainted?, Zstd.encode(src).tainted?)
-    src1 = src.dup
-    src1.taint
-    assert_same(src1.tainted?, Zstd.encode(src1).tainted?)
   end
 
   def test_huge
@@ -32,7 +27,15 @@ class TestZstd < Test::Unit::TestCase
     d.pos = 0
     size_b = 0
     md5b = Digest::MD5.new
-    Zstd.decode(d) { |z| buf = ""; while z.read(654321, buf); size_b += buf.bytesize; md5b.update buf; end }
+    partial_list = [262144, 1, 262144, 262142, 524288, 524280, 99, 999, 9999, 99999, 999999, 9999999, nil]
+    Zstd.decode(d) do |z|
+      buf = ""
+      while z.read(s = partial_list.shift, buf)
+        assert_equal s, buf.bytesize if s
+        size_b += buf.bytesize
+        md5b.update buf
+      end
+    end
     assert_equal size_a, size_b
     assert_equal(md5a.hexdigest, md5b.hexdigest)
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: extzstd
 version: !ruby/object:Gem::Version
-  version: '0.1'
+  version: 0.3.2
 platform: ruby
 authors:
 - dearblue
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-11-02 00:00:00.000000000 Z
+date: 2021-04-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -16,35 +16,39 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '12.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
-description: 'unoficial ruby bindings for Zstandard (zstd) <https://github.com/Cyan4973/zstd>.
+        version: '12.0'
+description: 'unoficial ruby bindings for Zstandard (zstd) <https://github.com/facebook/zstd>.
 
-'
+  '
 email: dearblue@users.osdn.me
 executables: []
 extensions:
 - ext/extconf.rb
 extra_rdoc_files:
-- HISTORY.ja
+- HISTORY.ja.md
 - LICENSE
 - README.md
+- contrib/zstd/CHANGELOG
 - contrib/zstd/LICENSE
 - contrib/zstd/README.md
+- contrib/zstd/lib/README.md
 - ext/extzstd.c
 - ext/extzstd.h
 - ext/extzstd_nogvls.h
 - ext/extzstd_stream.c
+- ext/libzstd_conf.h
 - ext/zstd_common.c
 - ext/zstd_compress.c
 - ext/zstd_decompress.c
 - ext/zstd_dictbuilder.c
+- ext/zstd_dictbuilder_fastcover.c
 - ext/zstd_legacy_v01.c
 - ext/zstd_legacy_v02.c
 - ext/zstd_legacy_v03.c
@@ -55,63 +59,116 @@ extra_rdoc_files:
 - lib/extzstd.rb
 - lib/extzstd/version.rb
 files:
-- HISTORY.ja
+- HISTORY.ja.md
 - LICENSE
 - README.md
 - Rakefile
+- contrib/zstd/CHANGELOG
+- contrib/zstd/CODE_OF_CONDUCT.md
+- contrib/zstd/CONTRIBUTING.md
+- contrib/zstd/COPYING
 - contrib/zstd/LICENSE
+- contrib/zstd/Makefile
 - contrib/zstd/README.md
-- contrib/zstd/common/bitstream.h
-- contrib/zstd/common/entropy_common.c
-- contrib/zstd/common/error_private.c
-- contrib/zstd/common/error_private.h
-- contrib/zstd/common/fse.h
-- contrib/zstd/common/fse_decompress.c
-- contrib/zstd/common/huf.h
-- contrib/zstd/common/mem.h
-- contrib/zstd/common/xxhash.c
-- contrib/zstd/common/xxhash.h
-- contrib/zstd/common/zbuff.h
-- contrib/zstd/common/zstd_common.c
-- contrib/zstd/common/zstd_errors.h
-- contrib/zstd/common/zstd_internal.h
-- contrib/zstd/compress/fse_compress.c
-- contrib/zstd/compress/huf_compress.c
-- contrib/zstd/compress/zbuff_compress.c
-- contrib/zstd/compress/zstd_compress.c
-- contrib/zstd/compress/zstd_opt.h
-- contrib/zstd/decompress/huf_decompress.c
-- contrib/zstd/decompress/zbuff_decompress.c
-- contrib/zstd/decompress/zstd_decompress.c
-- contrib/zstd/dictBuilder/divsufsort.c
-- contrib/zstd/dictBuilder/divsufsort.h
-- contrib/zstd/dictBuilder/zdict.c
-- contrib/zstd/dictBuilder/zdict.h
-- contrib/zstd/legacy/zstd_legacy.h
-- contrib/zstd/legacy/zstd_v01.c
-- contrib/zstd/legacy/zstd_v01.h
-- contrib/zstd/legacy/zstd_v02.c
-- contrib/zstd/legacy/zstd_v02.h
-- contrib/zstd/legacy/zstd_v03.c
-- contrib/zstd/legacy/zstd_v03.h
-- contrib/zstd/legacy/zstd_v04.c
-- contrib/zstd/legacy/zstd_v04.h
-- contrib/zstd/legacy/zstd_v05.c
-- contrib/zstd/legacy/zstd_v05.h
-- contrib/zstd/legacy/zstd_v06.c
-- contrib/zstd/legacy/zstd_v06.h
-- contrib/zstd/legacy/zstd_v07.c
-- contrib/zstd/legacy/zstd_v07.h
-- contrib/zstd/zstd.h
+- contrib/zstd/TESTING.md
+- contrib/zstd/appveyor.yml
+- contrib/zstd/lib/BUCK
+- contrib/zstd/lib/Makefile
+- contrib/zstd/lib/README.md
+- contrib/zstd/lib/common/bitstream.h
+- contrib/zstd/lib/common/compiler.h
+- contrib/zstd/lib/common/cpu.h
+- contrib/zstd/lib/common/debug.c
+- contrib/zstd/lib/common/debug.h
+- contrib/zstd/lib/common/entropy_common.c
+- contrib/zstd/lib/common/error_private.c
+- contrib/zstd/lib/common/error_private.h
+- contrib/zstd/lib/common/fse.h
+- contrib/zstd/lib/common/fse_decompress.c
+- contrib/zstd/lib/common/huf.h
+- contrib/zstd/lib/common/mem.h
+- contrib/zstd/lib/common/pool.c
+- contrib/zstd/lib/common/pool.h
+- contrib/zstd/lib/common/threading.c
+- contrib/zstd/lib/common/threading.h
+- contrib/zstd/lib/common/xxhash.c
+- contrib/zstd/lib/common/xxhash.h
+- contrib/zstd/lib/common/zstd_common.c
+- contrib/zstd/lib/common/zstd_deps.h
+- contrib/zstd/lib/common/zstd_errors.h
+- contrib/zstd/lib/common/zstd_internal.h
+- contrib/zstd/lib/compress/fse_compress.c
+- contrib/zstd/lib/compress/hist.c
+- contrib/zstd/lib/compress/hist.h
+- contrib/zstd/lib/compress/huf_compress.c
+- contrib/zstd/lib/compress/zstd_compress.c
+- contrib/zstd/lib/compress/zstd_compress_internal.h
+- contrib/zstd/lib/compress/zstd_compress_literals.c
+- contrib/zstd/lib/compress/zstd_compress_literals.h
+- contrib/zstd/lib/compress/zstd_compress_sequences.c
+- contrib/zstd/lib/compress/zstd_compress_sequences.h
+- contrib/zstd/lib/compress/zstd_compress_superblock.c
+- contrib/zstd/lib/compress/zstd_compress_superblock.h
+- contrib/zstd/lib/compress/zstd_cwksp.h
+- contrib/zstd/lib/compress/zstd_double_fast.c
+- contrib/zstd/lib/compress/zstd_double_fast.h
+- contrib/zstd/lib/compress/zstd_fast.c
+- contrib/zstd/lib/compress/zstd_fast.h
+- contrib/zstd/lib/compress/zstd_lazy.c
+- contrib/zstd/lib/compress/zstd_lazy.h
+- contrib/zstd/lib/compress/zstd_ldm.c
+- contrib/zstd/lib/compress/zstd_ldm.h
+- contrib/zstd/lib/compress/zstd_opt.c
+- contrib/zstd/lib/compress/zstd_opt.h
+- contrib/zstd/lib/compress/zstdmt_compress.c
+- contrib/zstd/lib/compress/zstdmt_compress.h
+- contrib/zstd/lib/decompress/huf_decompress.c
+- contrib/zstd/lib/decompress/zstd_ddict.c
+- contrib/zstd/lib/decompress/zstd_ddict.h
+- contrib/zstd/lib/decompress/zstd_decompress.c
+- contrib/zstd/lib/decompress/zstd_decompress_block.c
+- contrib/zstd/lib/decompress/zstd_decompress_block.h
+- contrib/zstd/lib/decompress/zstd_decompress_internal.h
+- contrib/zstd/lib/deprecated/zbuff.h
+- contrib/zstd/lib/deprecated/zbuff_common.c
+- contrib/zstd/lib/deprecated/zbuff_compress.c
+- contrib/zstd/lib/deprecated/zbuff_decompress.c
+- contrib/zstd/lib/dictBuilder/cover.c
+- contrib/zstd/lib/dictBuilder/cover.h
+- contrib/zstd/lib/dictBuilder/divsufsort.c
+- contrib/zstd/lib/dictBuilder/divsufsort.h
+- contrib/zstd/lib/dictBuilder/fastcover.c
+- contrib/zstd/lib/dictBuilder/zdict.c
+- contrib/zstd/lib/dictBuilder/zdict.h
+- contrib/zstd/lib/legacy/zstd_legacy.h
+- contrib/zstd/lib/legacy/zstd_v01.c
+- contrib/zstd/lib/legacy/zstd_v01.h
+- contrib/zstd/lib/legacy/zstd_v02.c
+- contrib/zstd/lib/legacy/zstd_v02.h
+- contrib/zstd/lib/legacy/zstd_v03.c
+- contrib/zstd/lib/legacy/zstd_v03.h
+- contrib/zstd/lib/legacy/zstd_v04.c
+- contrib/zstd/lib/legacy/zstd_v04.h
+- contrib/zstd/lib/legacy/zstd_v05.c
+- contrib/zstd/lib/legacy/zstd_v05.h
+- contrib/zstd/lib/legacy/zstd_v06.c
+- contrib/zstd/lib/legacy/zstd_v06.h
+- contrib/zstd/lib/legacy/zstd_v07.c
+- contrib/zstd/lib/legacy/zstd_v07.h
+- contrib/zstd/lib/libzstd.pc.in
+- contrib/zstd/lib/zstd.h
+- ext/depend
 - ext/extconf.rb
 - ext/extzstd.c
 - ext/extzstd.h
 - ext/extzstd_nogvls.h
 - ext/extzstd_stream.c
+- ext/libzstd_conf.h
 - ext/zstd_common.c
 - ext/zstd_compress.c
 - ext/zstd_decompress.c
 - ext/zstd_dictbuilder.c
+- ext/zstd_dictbuilder_fastcover.c
 - ext/zstd_legacy_v01.c
 - ext/zstd_legacy_v02.c
 - ext/zstd_legacy_v03.c
@@ -123,11 +180,11 @@ files:
 - lib/extzstd.rb
 - lib/extzstd/version.rb
 - test/test_basic.rb
-homepage: https://osdn.jp/projects/rutsubo/
+homepage: https://github.com/dearblue/ruby-extzstd/
 licenses:
-- 2-clause BSD License
+- BSD-2-Clause
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options:
 - "--charset"
 - UTF-8
@@ -146,9 +203,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.4
-signing_key: 
+rubygems_version: 3.2.14
+signing_key:
 specification_version: 4
 summary: ruby bindings for Zstandard (zstd)
 test_files: []

data/HISTORY.ja

@@ -1,5 +0,0 @@
-# extzstd の更新履歴
-
-## extzstd-0.1 (平成28年11月2日 (水曜日))
-
-初版

data/contrib/zstd/common/entropy_common.c

@@ -1,225 +0,0 @@
-/*
-   Common functions of New Generation Entropy library
-   Copyright (C) 2016, Yann Collet.
-
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
-   Redistribution and use in source and binary forms, with or without
-   modification, are permitted provided that the following conditions are
-   met:
-
-       * Redistributions of source code must retain the above copyright
-   notice, this list of conditions and the following disclaimer.
-       * Redistributions in binary form must reproduce the above
-   copyright notice, this list of conditions and the following disclaimer
-   in the documentation and/or other materials provided with the
-   distribution.
-
-   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-    You can contact the author at :
-    - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
-    - Public forum : https://groups.google.com/forum/#!forum/lz4c
-*************************************************************************** */
-
-/* *************************************
-*  Dependencies
-***************************************/
-#include "mem.h"
-#include "error_private.h"       /* ERR_*, ERROR */
-#define FSE_STATIC_LINKING_ONLY  /* FSE_MIN_TABLELOG */
-#include "fse.h"
-#define HUF_STATIC_LINKING_ONLY  /* HUF_TABLELOG_ABSOLUTEMAX */
-#include "huf.h"
-
-
-/*-****************************************
-*  FSE Error Management
-******************************************/
-unsigned FSE_isError(size_t code) { return ERR_isError(code); }
-
-const char* FSE_getErrorName(size_t code) { return ERR_getErrorName(code); }
-
-
-/* **************************************************************
-*  HUF Error Management
-****************************************************************/
-unsigned HUF_isError(size_t code) { return ERR_isError(code); }
-
-const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); }
-
-
-/*-**************************************************************
-*  FSE NCount encoding-decoding
-****************************************************************/
-static short FSE_abs(short a) { return (short)(a<0 ? -a : a); }
-
-size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
-                 const void* headerBuffer, size_t hbSize)
-{
-    const BYTE* const istart = (const BYTE*) headerBuffer;
-    const BYTE* const iend = istart + hbSize;
-    const BYTE* ip = istart;
-    int nbBits;
-    int remaining;
-    int threshold;
-    U32 bitStream;
-    int bitCount;
-    unsigned charnum = 0;
-    int previous0 = 0;
-
-    if (hbSize < 4) return ERROR(srcSize_wrong);
-    bitStream = MEM_readLE32(ip);
-    nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG;   /* extract tableLog */
-    if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge);
-    bitStream >>= 4;
-    bitCount = 4;
-    *tableLogPtr = nbBits;
-    remaining = (1<<nbBits)+1;
-    threshold = 1<<nbBits;
-    nbBits++;
-
-    while ((remaining>1) & (charnum<=*maxSVPtr)) {
-        if (previous0) {
-            unsigned n0 = charnum;
-            while ((bitStream & 0xFFFF) == 0xFFFF) {
-                n0 += 24;
-                if (ip < iend-5) {
-                    ip += 2;
-                    bitStream = MEM_readLE32(ip) >> bitCount;
-                } else {
-                    bitStream >>= 16;
-                    bitCount   += 16;
-            }   }
-            while ((bitStream & 3) == 3) {
-                n0 += 3;
-                bitStream >>= 2;
-                bitCount += 2;
-            }
-            n0 += bitStream & 3;
-            bitCount += 2;
-            if (n0 > *maxSVPtr) return ERROR(maxSymbolValue_tooSmall);
-            while (charnum < n0) normalizedCounter[charnum++] = 0;
-            if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) {
-                ip += bitCount>>3;
-                bitCount &= 7;
-                bitStream = MEM_readLE32(ip) >> bitCount;
-            } else {
-                bitStream >>= 2;
-        }   }
-        {   short const max = (short)((2*threshold-1)-remaining);
-            short count;
-
-            if ((bitStream & (threshold-1)) < (U32)max) {
-                count = (short)(bitStream & (threshold-1));
-                bitCount   += nbBits-1;
-            } else {
-                count = (short)(bitStream & (2*threshold-1));
-                if (count >= threshold) count -= max;
-                bitCount   += nbBits;
-            }
-
-            count--;   /* extra accuracy */
-            remaining -= FSE_abs(count);
-            normalizedCounter[charnum++] = count;
-            previous0 = !count;
-            while (remaining < threshold) {
-                nbBits--;
-                threshold >>= 1;
-            }
-
-            if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) {
-                ip += bitCount>>3;
-                bitCount &= 7;
-            } else {
-                bitCount -= (int)(8 * (iend - 4 - ip));
-                ip = iend - 4;
-            }
-            bitStream = MEM_readLE32(ip) >> (bitCount & 31);
-    }   }   /* while ((remaining>1) & (charnum<=*maxSVPtr)) */
-    if (remaining != 1) return ERROR(corruption_detected);
-    if (bitCount > 32) return ERROR(corruption_detected);
-    *maxSVPtr = charnum-1;
-
-    ip += (bitCount+7)>>3;
-    return ip-istart;
-}
-
-
-/*! HUF_readStats() :
-    Read compact Huffman tree, saved by HUF_writeCTable().
-    `huffWeight` is destination buffer.
-    @return : size read from `src` , or an error Code .
-    Note : Needed by HUF_readCTable() and HUF_readDTableX?() .
-*/
-size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
-                     U32* nbSymbolsPtr, U32* tableLogPtr,
-                     const void* src, size_t srcSize)
-{
-    U32 weightTotal;
-    const BYTE* ip = (const BYTE*) src;
-    size_t iSize;
-    size_t oSize;
-
-    if (!srcSize) return ERROR(srcSize_wrong);
-    iSize = ip[0];
-    /* memset(huffWeight, 0, hwSize);   *//* is not necessary, even though some analyzer complain ... */
-
-    if (iSize >= 128) {  /* special header */
-        oSize = iSize - 127;
-        iSize = ((oSize+1)/2);
-        if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
-        if (oSize >= hwSize) return ERROR(corruption_detected);
-        ip += 1;
-        {   U32 n;
-            for (n=0; n<oSize; n+=2) {
-                huffWeight[n]   = ip[n/2] >> 4;
-                huffWeight[n+1] = ip[n/2] & 15;
-    }   }   }
-    else  {   /* header compressed with FSE (normal case) */
-        if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
-        oSize = FSE_decompress(huffWeight, hwSize-1, ip+1, iSize);   /* max (hwSize-1) values decoded, as last one is implied */
-        if (FSE_isError(oSize)) return oSize;
-    }
-
-    /* collect weight stats */
-    memset(rankStats, 0, (HUF_TABLELOG_ABSOLUTEMAX + 1) * sizeof(U32));
-    weightTotal = 0;
-    {   U32 n; for (n=0; n<oSize; n++) {
-            if (huffWeight[n] >= HUF_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected);
-            rankStats[huffWeight[n]]++;
-            weightTotal += (1 << huffWeight[n]) >> 1;
-    }   }
-    if (weightTotal == 0) return ERROR(corruption_detected);
-
-    /* get last non-null symbol weight (implied, total must be 2^n) */
-    {   U32 const tableLog = BIT_highbit32(weightTotal) + 1;
-        if (tableLog > HUF_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected);
-        *tableLogPtr = tableLog;
-        /* determine last weight */
-        {   U32 const total = 1 << tableLog;
-            U32 const rest = total - weightTotal;
-            U32 const verif = 1 << BIT_highbit32(rest);
-            U32 const lastWeight = BIT_highbit32(rest) + 1;
-            if (verif != rest) return ERROR(corruption_detected);    /* last value must be a clean power of 2 */
-            huffWeight[oSize] = (BYTE)lastWeight;
-            rankStats[lastWeight]++;
-    }   }
-
-    /* check tree construction validity */
-    if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected);   /* by construction : at least 2 elts of rank 1, must be even */
-
-    /* results */
-    *nbSymbolsPtr = (U32)(oSize+1);
-    return iSize+1;
-}