extzstd 0.0.2.CONCEPT → 0.0.3.CONCEPT

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e5563734d016a497235ac43fed1b14a41d32e1c3
-  data.tar.gz: 63389454aa8f607e5f823b27314be3ae7efe45ed
+  metadata.gz: 2fe6b36e5eb4cffa0d077511dcbd70d869d499f1
+  data.tar.gz: 9acfaccd4a530eff2581bb407a2d221a1926e3e1
 SHA512:
-  metadata.gz: a7dd05b9080beec1707db4e76948154309052fb85c615a211bcb4c82147fc84a666aadeddf670dbb1932a406ef88448e133ed33967cd06b27af2f1536c6ab042
-  data.tar.gz: 53e088c35e916c5bb21e10fe849e57a20e1e933b9f97cc5ad43be6c0e8531fe3984682bbeb2fe7b542fadccffb07ac9e2b1a8a978696fd43ba80a6155d7bbc6c
+  metadata.gz: 23b1a8f69661f30443e2eeb0207dc4260b760e2cff9240fcc8e7f4f89656d12f3302f983c9101a4f5fb786304e61d854fa83f36199640a082950a292b6eeb47f
+  data.tar.gz: 0c666f160c2d2a53c52327bbc1e482152f712c11eaff45ae6e472467c84f6a8be96d2e716d5034cc9bf3ce32491ff4f045c96b897458f45653e9f99b5c68a1bc

data/README.md

@@ -1,38 +1,53 @@
-# encoding:utf-8 ;
-
 # extzstd - ruby bindings for Zstd (Zstandard)
 
-This is ruby bindings for compression library
+This is unofficial ruby bindings for compression library
 [Zstd (https://github.com/Cyan4973/zstd)](https://github.com/Cyan4973/zstd).
 
   * package name: extzstd
-  * author: dearblue (mailto:dearblue@users.osdn.me)
-  * version: 0.0.2.CONCEPT
+  * version: 0.0.3.CONCEPT
   * software quality: EXPERIMENTAL
   * license: 2-clause BSD License
-  * report issue to: https://osdn.jp/projects/rutsubo/ticket/
+  * author: dearblue <mailto:dearblue@users.osdn.me>
+  * report issue to: <https://osdn.jp/projects/rutsubo/ticket/>
   * dependency ruby: ruby-2.0+
   * dependency ruby gems: (none)
   * dependency library: (none)
-  * bundled external libraries:
-      * zstd-0.1.2 (https://github.com/Cyan4973/zstd/tree/zstd-0.1.2)
+  * bundled external C library:
+      * zstd-0.7.4 <https://github.com/Cyan4973/zstd/tree/v0.7.4>
+
+"extzstd" is supported the legacy formats (zstd-0.1 - 0.6).
 
 
-## ***WARNING***
+## Quick API Reference
 
-Zstd data format compatibility is not guaranteed in future versions
-(There is a possibility that it becomes impossible to future use).
+  * Encode
+      * ``Zstd.encode(buf, params = nil, dict = nil) -> encoded string``
+      * ``Zstd.encode(outport, params = nil, dict = nil) -> an instance of Zstd::Encoder``
+      * ``Zstd.encode(outport, params = nil, dict = nil) { |encoder| ... } -> block returned value``
+      * ``Zstd::Encoder#write(buf) -> this instance``
+      * ``Zstd::Encoder#close -> nil``
 
-Written in [zstd/README.md](https://github.com/Cyan4973/zstd/blob/zstd-0.1.2/README.md):
+  * Decode
+      * ``Zstd.decode(zstd_buf, dict = nil) -> decoded string``
+      * ``Zstd.decode(inport, dict = nil) -> an intance of Zstd::Decoder``
+      * ``Zstd.decode(inport, dict = nil) { |decoder| ... } -> block returned value``
+      * ``Zstd::Decoder#read(size = nil, buf = nil) -> buf``
+      * ``Zstd::Decoder#close -> nil``
 
->   Zstd has not yet reached "stable" status. Specifically, it doesn't
->   guarantee yet that its current compressed format will remain stable
->   and supported in future versions.
+  * buffered encoder
+      * ``Zstd::BufferedEncoder.new(params, dict)`` (``ZBUFF_createCCtx``, ``ZBUFF_compressInit_advanced``)
+      * ``Zstd::BufferedEncoder#continue(data, dataoff, dest, maxdest) -> integer as new data offset`` (``ZBUFF_compressContinue``)
+      * ``Zstd::BufferedEncoder#flush(dest, maxdest) -> dest`` (``ZBUFF_compressFlush``)
+      * ``Zstd::BufferedEncoder#end(dest, maxdest) -> dest`` (``ZBUFF_compressEnd``)
+
+  * Train dictionary
+      * ``Zstd.dict_train_from_buffer(buf, dict_capacity) -> dictionary'd string`` (``ZDICT_trainFromBuffer``)
+      * ``Zstd.dict_add_entropy_tables_from_buffer(dict, dict_capacity, sample) -> dict`` (``ZDICT_addEntropyTablesFromBuffer``)
 
 
 ## HOW TO USE
 
-### basic usage (one pass encode/decode)
+### basic usage (simply encode/decode)
 
 ``` ruby:ruby
 # First, load library
@@ -41,19 +56,54 @@ require "extzstd"
 # Prepair data
 source = "sample data..." * 50
 
-# Directly compression
+# simply compression
 encdata = Zstd.encode(source)
 puts "encdata.bytesize=#{encdata.bytesize}"
 
-# Directly decompression
-maxdestsize = source.bytesize
-decdata = Zstd.decode(encdata, maxdestsize)
+# simply decompression
+decdata = Zstd.decode(encdata)
 puts "decdata.bytesize=#{decdata.bytesize}"
 
 # Comparison source and decoded data
 p source == decdata # => true
 ```
 
-----
+### Streaming compression (with block)
+
+``` ruby:ruby
+outport = StringIO.new("")
+Zstd.encode(outport) do |encoder|
+  encoder.write "abcdefg\n"
+  encoder << "hijklmn\n"
+  encoder.write "opqrstu\n"
+  encoder << "vwxyz\n"
+end
+```
+
+### Streaming compression (without block and write to file)
 
-[a stub]
+``` ruby:ruby
+file = File.open("sample.zst", "wb")
+encoder = Zstd.encode(file)
+
+encoder.write "abcdefg\n"
+encoder << "hijklmn\n"
+encoder.write "opqrstu\n"
+encoder << "vwxyz\n"
+
+encoder.close
+file.close
+```
+
+### Streaming decompression (with block and read from file)
+
+``` ruby:ruby
+File.open("sample.zst", "rb") do |file|
+  Zstd.decode(file) do |decoder|
+    p decoder.read(8)  # => "abcdefg\n"
+    p decoder.read(1)  # => "h"
+    p decoder.read(2)  # => "ij"
+    p decoder.read     # => "klmn\nopqrstu\nvwxyz\n"
+  end
+end
+```

data/Rakefile

@@ -1,11 +1,14 @@
 
+require "pathname"
 require "rake/clean"
 
-DOC = FileList["{README,LICENSE,CHANGELOG,Changelog,HISTORY}{,.ja}{,.txt,.rd,.rdoc,.md,.markdown}"] +
-      FileList["{contrib,ext}/**/{README,LICENSE,CHANGELOG,Changelog,HISTORY}{,.ja}{,.txt,.rd,.rdoc,.md,.markdown}"] +
-      FileList["ext/**/*.{c,C,cc,cxx,cpp,h,H,hh}"]
-#EXT = FileList["ext/**/*.{h,hh,c,cc,cpp,cxx}"] +
-#      FileList["ext/externals/**/*"]
+docnames = "{README,LICENSE,CHANGELOG,Changelog,HISTORY}"
+doctypes = "{,.txt,.rd,.rdoc,.md,.markdown}"
+cexttypes = "{c,C,cc,cxx,cpp,h,H,hh}"
+
+DOC = FileList["#{docnames}{,.ja}#{doctypes}"] +
+      FileList["{contrib,ext}/**/#{docnames}{,.ja}#{doctypes}"] +
+      FileList["ext/**/*.#{cexttypes}"]
 EXT = FileList["ext/**/*"]
 BIN = FileList["bin/*"]
 LIB = FileList["lib/**/*.rb"]
@@ -15,18 +18,23 @@ EXAMPLE = FileList["examples/**/*"]
 GEMSTUB_SRC = "gemstub.rb"
 RAKEFILE = [File.basename(__FILE__), GEMSTUB_SRC]
 EXTRA = []
+EXTCONF = FileList["ext/**/extconf.rb"]
+EXTCONF.reject! { |n| !File.file?(n) }
+EXTMAP = {}
 
 load GEMSTUB_SRC
 
-EXTCONF = FileList["ext/extconf.rb"]
-EXTCONF.reject! { |n| !File.file?(n) }
+EXTMAP.dup.each_pair do |dir, name|
+  EXTMAP[Pathname.new(dir).cleanpath.to_s] = Pathname.new(name).cleanpath.to_s
+end
+
 GEMSTUB.extensions += EXTCONF
 GEMSTUB.executables += FileList["bin/*"].map { |n| File.basename n }
 GEMSTUB.executables.sort!
 
 PACKAGENAME = "#{GEMSTUB.name}-#{GEMSTUB.version}"
 GEMFILE = "#{PACKAGENAME}.gem"
-GEMSPEC = "#{GEMSTUB.name}.gemspec"
+GEMSPEC = "#{PACKAGENAME}.gemspec"
 
 GEMSTUB.files += DOC + EXT + EXTCONF + BIN + LIB + SPEC + TEST + EXAMPLE + RAKEFILE + EXTRA
 GEMSTUB.files.sort!
@@ -37,11 +45,27 @@ end
 GEMSTUB.extra_rdoc_files += DOC + LIB + EXT.reject { |n| n.include?("/externals/") || !%w(.h .hh .c .cc .cpp .cxx).include?(File.extname(n)) }
 GEMSTUB.extra_rdoc_files.sort!
 
-CLEAN << GEMSPEC
+GEMSTUB_TRYOUT = GEMSTUB.dup
+GEMSTUB_TRYOUT.version = "#{GEMSTUB.version}#{Time.now.strftime(".TRYOUT.%Y%m%d.%H%M%S")}"
+PACKAGENAME_TRYOUT = "#{GEMSTUB.name}-#{GEMSTUB_TRYOUT.version}"
+GEMFILE_TRYOUT = "#{PACKAGENAME_TRYOUT}.gem"
+GEMSPEC_TRYOUT = "#{PACKAGENAME_TRYOUT}.gemspec"
+
+CLEAN << GEMSPEC << GEMSPEC_TRYOUT
 CLOBBER << GEMFILE
 
-task :default => :all
+task :default => :tryout do
+  $stderr.puts <<-EOS
+#{__FILE__}:#{__LINE__}:
+\ttype ``rake release'' to build release package.
+  EOS
+end
+
+desc "build tryout package"
+task :tryout
 
+desc "build release package"
+task :release => :all
 
 unless EXTCONF.empty?
   RUBYSET ||= (ENV["RUBYSET"] || "").split(",")
@@ -54,24 +78,43 @@ unless EXTCONF.empty?
 | variable for set ruby interpreters by comma separated.
 |
 |   e.g.) $ rake RUBYSET=ruby
-|     or) $ rake RUBYSET=ruby20,ruby21,ruby22
+|     or) $ rake RUBYSET=ruby21,ruby22,ruby23
 |
     EOS
   else
-    platforms = RUBYSET.map { |ruby| `#{ruby} --disable gems -rrbconfig -e "puts RbConfig::CONFIG['arch']"`.chomp }
+    platforms = RUBYSET.map { |ruby| `#{ruby} --disable-gems -e "puts RUBY_PLATFORM"`.chomp }
     platforms1 = platforms.uniq
     unless platforms1.size == 1 && !platforms1[0].empty?
-      raise "different platforms (#{Hash[*RUBYSET.zip(platforms).flatten].inspect})"
+      abort <<-EOS
+#{__FILE__}:#{__LINE__}: different platforms:
+#{RUBYSET.zip(platforms).map { |ruby, platform| "%24s => %s" % [ruby, platform] }.join("\n")}
+ABORTED.
+      EOS
     end
     PLATFORM = platforms1[0]
 
     RUBY_VERSIONS = RUBYSET.map do |ruby|
-      ver = `#{ruby} --disable gem -rrbconfig -e "puts RbConfig::CONFIG['ruby_version']"`.slice(/\d+\.\d+/)
+      ver = `#{ruby} --disable-gems -e "puts RUBY_VERSION"`.slice(/\d+\.\d+/)
       raise "failed ruby checking - ``#{ruby}''" unless $?.success?
       [ver, ruby]
     end
-    SOFILES_SET = RUBY_VERSIONS.map { |(ver, ruby)| ["lib/#{ver}/#{GEMSTUB.name}.so", ruby] }
-    SOFILES = SOFILES_SET.map { |(lib, ruby)| lib }
+
+    SOFILES_SET = RUBY_VERSIONS.map { |(ver, ruby)|
+      EXTCONF.map { |extconf|
+        extdir = Pathname.new(extconf).cleanpath.dirname.to_s
+        case
+        when soname = EXTMAP[extdir.sub(/^ext\//i, "")]
+          soname = soname.sub(/\.so$/i, "")
+        when extdir == "ext" || extdir == "."
+          soname = GEMSTUB.name
+        else
+          soname = File.basename(extdir)
+        end
+
+        [ruby, File.join("lib", "#{soname.sub(/(?<=\/)|^(?!.*\/)/, "#{ver}/")}.so"), extconf]
+      }
+    }.flatten(1)
+    SOFILES = SOFILES_SET.map { |(ruby, sopath, extconf)| sopath }
 
     GEMSTUB_NATIVE = GEMSTUB.dup
     GEMSTUB_NATIVE.files += SOFILES
@@ -99,7 +142,7 @@ unless EXTCONF.empty?
     desc "build c-extension libraries"
     task "sofiles" => SOFILES
 
-    SOFILES_SET.each do |(soname, ruby)|
+    SOFILES_SET.each do |(ruby, soname, extconf)|
       sodir = File.dirname(soname)
       makefile = File.join(sodir, "Makefile")
 
@@ -109,9 +152,10 @@ unless EXTCONF.empty?
       directory sodir
 
       desc "generate Makefile for binary extension library"
-      file makefile => [sodir] + EXTCONF do
+      file makefile => [sodir, extconf] do
+        rel_extconf = Pathname.new(extconf).relative_path_from(Pathname.new(sodir)).to_s
         cd sodir do
-          sh "#{ruby} ../../#{EXTCONF[0]} \"--ruby=#{ruby}\""
+          sh *%W"#{ruby} #{rel_extconf} --ruby=#{ruby} #{ENV["EXTCONF"]}"
         end
       end
 
@@ -127,6 +171,7 @@ end
 
 
 task :all => GEMFILE
+task :tryout => GEMFILE_TRYOUT
 
 desc "generate local rdoc"
 task :rdoc => DOC + LIB do
@@ -153,6 +198,15 @@ file GEMFILE => DOC + EXT + EXTCONF + BIN + LIB + SPEC + TEST + EXAMPLE + RAKEFI
   sh "gem build #{GEMSPEC}"
 end
 
+file GEMFILE_TRYOUT => DOC + EXT + EXTCONF + BIN + LIB + SPEC + TEST + EXAMPLE + RAKEFILE + [GEMSPEC_TRYOUT] do
+#file GEMFILE_TRYOUT do
+  sh "gem build #{GEMSPEC_TRYOUT}"
+end
+
 file GEMSPEC => RAKEFILE do
   File.write(GEMSPEC, GEMSTUB.to_ruby, mode: "wb")
 end
+
+file GEMSPEC_TRYOUT => RAKEFILE do
+  File.write(GEMSPEC_TRYOUT, GEMSTUB_TRYOUT.to_ruby, mode: "wb")
+end

data/contrib/zstd/README.md

@@ -0,0 +1,68 @@
+zstd - library files
+================================
+
+The __lib__ directory contains several files, but depending on target use case, some of them may not be necessary.
+
+#### Minimal library files
+
+To build the zstd library the following files are required:
+
+- [common/bitstream.h](common/bitstream.h)
+- [common/error_private.h](common/error_private.h)
+- [common/error_public.h](common/error_public.h)
+- common/fse.h
+- common/fse_decompress.c
+- common/huf.h
+- [common/mem.h](common/mem.h)
+- [common/zstd.h]
+- common/zstd_internal.h
+- compress/fse_compress.c
+- compress/huf_compress.c
+- compress/zstd_compress.c
+- compress/zstd_opt.h
+- decompress/huf_decompress.c
+- decompress/zstd_decompress.c
+
+Stable API is exposed in [common/zstd.h].
+Advanced and experimental API can be enabled by defining `ZSTD_STATIC_LINKING_ONLY`.
+Never use them with a dynamic library, as their definition may change in future versions.
+
+[common/zstd.h]: common/zstd.h
+
+
+#### Separate compressor and decompressor
+
+To build a separate zstd compressor all files from `common/` and `compressor/` directories are required.
+In a similar way to build a separate zstd decompressor all files from `common/` and `decompressor/` directories are needed.
+
+
+#### Buffered streaming
+
+This complementary API makes streaming integration easier.
+It is used by `zstd` command line utility, and [7zip plugin](http://mcmilk.de/projects/7-Zip-ZStd) :
+
+- common/zbuff.h
+- compress/zbuff_compress.c
+- decompress/zbuff_decompress.c
+
+
+#### Dictionary builder
+
+In order to create dictionaries from some training sets,
+it's needed to include all files from [dictBuilder directory](dictBuilder/)
+
+
+#### Legacy support
+
+Zstandard can decode previous formats, starting from v0.1.
+Support for these format is provided in [folder legacy](legacy/).
+It's also required to compile the library with `ZSTD_LEGACY_SUPPORT = 1`.
+
+
+#### Miscellaneous
+
+The other files are not source code. There are :
+
+ - LICENSE : contains the BSD license text
+ - Makefile : script to compile or install zstd library (static or dynamic)
+ - libzstd.pc.in : for pkg-config (make install)

data/contrib/zstd/common/bitstream.h

@@ -0,0 +1,414 @@
+/* ******************************************************************
+   bitstream
+   Part of FSE library
+   header file (to include)
+   Copyright (C) 2013-2016, Yann Collet.
+
+   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions are
+   met:
+
+       * Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+       * Redistributions in binary form must reproduce the above
+   copyright notice, this list of conditions and the following disclaimer
+   in the documentation and/or other materials provided with the
+   distribution.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+   You can contact the author at :
+   - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
+****************************************************************** */
+#ifndef BITSTREAM_H_MODULE
+#define BITSTREAM_H_MODULE
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+
+/*
+*  This API consists of small unitary functions, which must be inlined for best performance.
+*  Since link-time-optimization is not available for all compilers,
+*  these functions are defined into a .h to be included.
+*/
+
+/*-****************************************
+*  Dependencies
+******************************************/
+#include "mem.h"            /* unaligned access routines */
+#include "error_private.h"  /* error codes and messages */
+
+
+/*=========================================
+*  Target specific
+=========================================*/
+#if defined(__BMI__) && defined(__GNUC__)
+#  include <immintrin.h>   /* support for bextr (experimental) */
+#endif
+
+
+/*-******************************************
+*  bitStream encoding API (write forward)
+********************************************/
+/* bitStream can mix input from multiple sources.
+*  A critical property of these streams is that they encode and decode in **reverse** direction.
+*  So the first bit sequence you add will be the last to be read, like a LIFO stack.
+*/
+typedef struct
+{
+    size_t bitContainer;
+    int    bitPos;
+    char*  startPtr;
+    char*  ptr;
+    char*  endPtr;
+} BIT_CStream_t;
+
+MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* dstBuffer, size_t dstCapacity);
+MEM_STATIC void   BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits);
+MEM_STATIC void   BIT_flushBits(BIT_CStream_t* bitC);
+MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC);
+
+/* Start with initCStream, providing the size of buffer to write into.
+*  bitStream will never write outside of this buffer.
+*  `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code.
+*
+*  bits are first added to a local register.
+*  Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems.
+*  Writing data into memory is an explicit operation, performed by the flushBits function.
+*  Hence keep track how many bits are potentially stored into local register to avoid register overflow.
+*  After a flushBits, a maximum of 7 bits might still be stored into local register.
+*
+*  Avoid storing elements of more than 24 bits if you want compatibility with 32-bits bitstream readers.
+*
+*  Last operation is to close the bitStream.
+*  The function returns the final size of CStream in bytes.
+*  If data couldn't fit into `dstBuffer`, it will return a 0 ( == not storable)
+*/
+
+
+/*-********************************************
+*  bitStream decoding API (read backward)
+**********************************************/
+typedef struct
+{
+    size_t   bitContainer;
+    unsigned bitsConsumed;
+    const char* ptr;
+    const char* start;
+} BIT_DStream_t;
+
+typedef enum { BIT_DStream_unfinished = 0,
+               BIT_DStream_endOfBuffer = 1,
+               BIT_DStream_completed = 2,
+               BIT_DStream_overflow = 3 } BIT_DStream_status;  /* result of BIT_reloadDStream() */
+               /* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */
+
+MEM_STATIC size_t   BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize);
+MEM_STATIC size_t   BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits);
+MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD);
+MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
+
+
+/* Start by invoking BIT_initDStream().
+*  A chunk of the bitStream is then stored into a local register.
+*  Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
+*  You can then retrieve bitFields stored into the local register, **in reverse order**.
+*  Local register is explicitly reloaded from memory by the BIT_reloadDStream() method.
+*  A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished.
+*  Otherwise, it can be less than that, so proceed accordingly.
+*  Checking if DStream has reached its end can be performed with BIT_endOfDStream().
+*/
+
+
+/*-****************************************
+*  unsafe API
+******************************************/
+MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits);
+/* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */
+
+MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC);
+/* unsafe version; does not check buffer overflow */
+
+MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
+/* faster, but works only if nbBits >= 1 */
+
+
+
+/*-**************************************************************
+*  Internal functions
+****************************************************************/
+MEM_STATIC unsigned BIT_highbit32 (register U32 val)
+{
+#   if defined(_MSC_VER)   /* Visual */
+    unsigned long r=0;
+    _BitScanReverse ( &r, val );
+    return (unsigned) r;
+#   elif defined(__GNUC__) && (__GNUC__ >= 3)   /* Use GCC Intrinsic */
+    return 31 - __builtin_clz (val);
+#   else   /* Software version */
+    static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
+    U32 v = val;
+    v |= v >> 1;
+    v |= v >> 2;
+    v |= v >> 4;
+    v |= v >> 8;
+    v |= v >> 16;
+    return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27];
+#   endif
+}
+
+/*=====    Local Constants   =====*/
+static const unsigned BIT_mask[] = { 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF,  0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF };   /* up to 26 bits */
+
+
+/*-**************************************************************
+*  bitStream encoding
+****************************************************************/
+/*! BIT_initCStream() :
+ *  `dstCapacity` must be > sizeof(void*)
+ *  @return : 0 if success,
+              otherwise an error code (can be tested using ERR_isError() ) */
+MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* startPtr, size_t dstCapacity)
+{
+    bitC->bitContainer = 0;
+    bitC->bitPos = 0;
+    bitC->startPtr = (char*)startPtr;
+    bitC->ptr = bitC->startPtr;
+    bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->ptr);
+    if (dstCapacity <= sizeof(bitC->ptr)) return ERROR(dstSize_tooSmall);
+    return 0;
+}
+
+/*! BIT_addBits() :
+    can add up to 26 bits into `bitC`.
+    Does not check for register overflow ! */
+MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits)
+{
+    bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos;
+    bitC->bitPos += nbBits;
+}
+
+/*! BIT_addBitsFast() :
+ *  works only if `value` is _clean_, meaning all high bits above nbBits are 0 */
+MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits)
+{
+    bitC->bitContainer |= value << bitC->bitPos;
+    bitC->bitPos += nbBits;
+}
+
+/*! BIT_flushBitsFast() :
+ *  unsafe version; does not check buffer overflow */
+MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC)
+{
+    size_t const nbBytes = bitC->bitPos >> 3;
+    MEM_writeLEST(bitC->ptr, bitC->bitContainer);
+    bitC->ptr += nbBytes;
+    bitC->bitPos &= 7;
+    bitC->bitContainer >>= nbBytes*8;   /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */
+}
+
+/*! BIT_flushBits() :
+ *  safe version; check for buffer overflow, and prevents it.
+ *  note : does not signal buffer overflow. This will be revealed later on using BIT_closeCStream() */
+MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC)
+{
+    size_t const nbBytes = bitC->bitPos >> 3;
+    MEM_writeLEST(bitC->ptr, bitC->bitContainer);
+    bitC->ptr += nbBytes;
+    if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr;
+    bitC->bitPos &= 7;
+    bitC->bitContainer >>= nbBytes*8;   /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */
+}
+
+/*! BIT_closeCStream() :
+ *  @return : size of CStream, in bytes,
+              or 0 if it could not fit into dstBuffer */
+MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC)
+{
+    BIT_addBitsFast(bitC, 1, 1);   /* endMark */
+    BIT_flushBits(bitC);
+
+    if (bitC->ptr >= bitC->endPtr) return 0; /* doesn't fit within authorized budget : cancel */
+
+    return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0);
+}
+
+
+/*-********************************************************
+* bitStream decoding
+**********************************************************/
+/*! BIT_initDStream() :
+*   Initialize a BIT_DStream_t.
+*   `bitD` : a pointer to an already allocated BIT_DStream_t structure.
+*   `srcSize` must be the *exact* size of the bitStream, in bytes.
+*   @return : size of stream (== srcSize) or an errorCode if a problem is detected
+*/
+MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize)
+{
+    if (srcSize < 1) { memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); }
+
+    if (srcSize >=  sizeof(bitD->bitContainer)) {  /* normal case */
+        bitD->start = (const char*)srcBuffer;
+        bitD->ptr   = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer);
+        bitD->bitContainer = MEM_readLEST(bitD->ptr);
+        { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
+          bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0;
+          if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ }
+    } else {
+        bitD->start = (const char*)srcBuffer;
+        bitD->ptr   = bitD->start;
+        bitD->bitContainer = *(const BYTE*)(bitD->start);
+        switch(srcSize)
+        {
+            case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16);
+            case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24);
+            case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32);
+            case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24;
+            case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16;
+            case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) <<  8;
+            default:;
+        }
+        { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
+          bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0;
+          if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ }
+        bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8;
+    }
+
+    return srcSize;
+}
+
+MEM_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start)
+{
+    return bitContainer >> start;
+}
+
+MEM_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits)
+{
+#if defined(__BMI__) && defined(__GNUC__)   /* experimental */
+#  if defined(__x86_64__)
+    if (sizeof(bitContainer)==8)
+        return _bextr_u64(bitContainer, start, nbBits);
+    else
+#  endif
+        return _bextr_u32(bitContainer, start, nbBits);
+#else
+    return (bitContainer >> start) & BIT_mask[nbBits];
+#endif
+}
+
+MEM_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits)
+{
+    return bitContainer & BIT_mask[nbBits];
+}
+
+/*! BIT_lookBits() :
+ *  Provides next n bits from local register.
+ *  local register is not modified.
+ *  On 32-bits, maxNbBits==24.
+ *  On 64-bits, maxNbBits==56.
+ *  @return : value extracted
+ */
+ MEM_STATIC size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits)
+{
+#if defined(__BMI__) && defined(__GNUC__)   /* experimental; fails if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8 */
+    return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits);
+#else
+    U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1;
+    return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask);
+#endif
+}
+
+/*! BIT_lookBitsFast() :
+*   unsafe version; only works only if nbBits >= 1 */
+MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits)
+{
+    U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1;
+    return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask+1)-nbBits) & bitMask);
+}
+
+MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
+{
+    bitD->bitsConsumed += nbBits;
+}
+
+/*! BIT_readBits() :
+ *  Read (consume) next n bits from local register and update.
+ *  Pay attention to not read more than nbBits contained into local register.
+ *  @return : extracted value.
+ */
+MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
+{
+    size_t const value = BIT_lookBits(bitD, nbBits);
+    BIT_skipBits(bitD, nbBits);
+    return value;
+}
+
+/*! BIT_readBitsFast() :
+*   unsafe version; only works only if nbBits >= 1 */
+MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
+{
+    size_t const value = BIT_lookBitsFast(bitD, nbBits);
+    BIT_skipBits(bitD, nbBits);
+    return value;
+}
+
+/*! BIT_reloadDStream() :
+*   Refill `BIT_DStream_t` from src buffer previously defined (see BIT_initDStream() ).
+*   This function is safe, it guarantees it will not read beyond src buffer.
+*   @return : status of `BIT_DStream_t` internal register.
+              if status == unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */
+MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
+{
+	if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8))  /* should not happen => corruption detected */
+		return BIT_DStream_overflow;
+
+    if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) {
+        bitD->ptr -= bitD->bitsConsumed >> 3;
+        bitD->bitsConsumed &= 7;
+        bitD->bitContainer = MEM_readLEST(bitD->ptr);
+        return BIT_DStream_unfinished;
+    }
+    if (bitD->ptr == bitD->start) {
+        if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer;
+        return BIT_DStream_completed;
+    }
+    {   U32 nbBytes = bitD->bitsConsumed >> 3;
+        BIT_DStream_status result = BIT_DStream_unfinished;
+        if (bitD->ptr - nbBytes < bitD->start) {
+            nbBytes = (U32)(bitD->ptr - bitD->start);  /* ptr > start */
+            result = BIT_DStream_endOfBuffer;
+        }
+        bitD->ptr -= nbBytes;
+        bitD->bitsConsumed -= nbBytes*8;
+        bitD->bitContainer = MEM_readLEST(bitD->ptr);   /* reminder : srcSize > sizeof(bitD) */
+        return result;
+    }
+}
+
+/*! BIT_endOfDStream() :
+*   @return Tells if DStream has exactly reached its end (all bits consumed).
+*/
+MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)
+{
+    return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8));
+}
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* BITSTREAM_H_MODULE */