extzstd 0.0.2.CONCEPT-x86-mingw32 → 0.0.3.CONCEPT-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.
Files changed (72) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +72 -22
  3. data/Rakefile +73 -19
  4. data/contrib/zstd/README.md +68 -0
  5. data/contrib/zstd/common/bitstream.h +414 -0
  6. data/contrib/zstd/common/entropy_common.c +231 -0
  7. data/contrib/zstd/common/error_private.h +125 -0
  8. data/contrib/zstd/common/error_public.h +77 -0
  9. data/contrib/zstd/common/fse.h +628 -0
  10. data/contrib/zstd/common/fse_decompress.c +331 -0
  11. data/contrib/zstd/common/huf.h +228 -0
  12. data/contrib/zstd/common/mem.h +377 -0
  13. data/contrib/zstd/common/xxhash.c +854 -0
  14. data/contrib/zstd/common/xxhash.h +273 -0
  15. data/contrib/zstd/common/zbuff.h +197 -0
  16. data/contrib/zstd/common/zstd.h +475 -0
  17. data/contrib/zstd/common/zstd_common.c +91 -0
  18. data/contrib/zstd/common/zstd_internal.h +238 -0
  19. data/contrib/zstd/compress/fse_compress.c +807 -0
  20. data/contrib/zstd/compress/huf_compress.c +577 -0
  21. data/contrib/zstd/compress/zbuff_compress.c +327 -0
  22. data/contrib/zstd/compress/zstd_compress.c +3074 -0
  23. data/contrib/zstd/compress/zstd_opt.h +1046 -0
  24. data/contrib/zstd/decompress/huf_decompress.c +894 -0
  25. data/contrib/zstd/decompress/zbuff_decompress.c +294 -0
  26. data/contrib/zstd/decompress/zstd_decompress.c +1362 -0
  27. data/contrib/zstd/dictBuilder/divsufsort.c +1913 -0
  28. data/contrib/zstd/dictBuilder/divsufsort.h +67 -0
  29. data/contrib/zstd/dictBuilder/zdict.c +1045 -0
  30. data/contrib/zstd/dictBuilder/zdict.h +113 -0
  31. data/contrib/zstd/legacy/zstd_legacy.h +140 -0
  32. data/contrib/zstd/legacy/zstd_v01.c +2178 -0
  33. data/contrib/zstd/{zstd.h → legacy/zstd_v01.h} +46 -39
  34. data/contrib/zstd/legacy/zstd_v02.c +3748 -0
  35. data/contrib/zstd/legacy/zstd_v02.h +99 -0
  36. data/contrib/zstd/legacy/zstd_v03.c +3389 -0
  37. data/contrib/zstd/legacy/zstd_v03.h +99 -0
  38. data/contrib/zstd/legacy/zstd_v04.c +4056 -0
  39. data/contrib/zstd/legacy/zstd_v04.h +148 -0
  40. data/contrib/zstd/legacy/zstd_v05.c +4325 -0
  41. data/contrib/zstd/legacy/zstd_v05.h +171 -0
  42. data/contrib/zstd/legacy/zstd_v06.c +4581 -0
  43. data/contrib/zstd/legacy/zstd_v06.h +185 -0
  44. data/ext/extconf.rb +10 -12
  45. data/ext/extzstd.c +497 -144
  46. data/ext/extzstd.h +127 -22
  47. data/ext/extzstd_buffered.c +265 -0
  48. data/ext/extzstd_nogvls.h +174 -0
  49. data/ext/zstd_amalgam.c +18 -0
  50. data/ext/zstd_legacy_v01.c +1 -0
  51. data/ext/zstd_legacy_v02.c +1 -0
  52. data/ext/zstd_legacy_v03.c +1 -0
  53. data/ext/zstd_legacy_v04.c +1 -0
  54. data/ext/zstd_legacy_v05.c +1 -0
  55. data/ext/zstd_legacy_v06.c +1 -0
  56. data/gemstub.rb +17 -1
  57. data/lib/2.0/extzstd.so +0 -0
  58. data/lib/2.1/extzstd.so +0 -0
  59. data/lib/2.2/extzstd.so +0 -0
  60. data/lib/2.3/extzstd.so +0 -0
  61. data/lib/extzstd.rb +197 -77
  62. data/lib/extzstd/version.rb +1 -1
  63. data/test/test_basic.rb +41 -0
  64. metadata +71 -20
  65. data/contrib/zstd/Makefile +0 -115
  66. data/contrib/zstd/fse.c +0 -2466
  67. data/contrib/zstd/fse.h +0 -320
  68. data/contrib/zstd/fse_static.h +0 -282
  69. data/contrib/zstd/libzstd.pc.in +0 -14
  70. data/contrib/zstd/zstd.c +0 -1768
  71. data/contrib/zstd/zstd_static.h +0 -89
  72. data/ext/extzstd-stream.c +0 -398
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 193b3a5ca32c1263df30b3d75266ca13981424b4
4
- data.tar.gz: 6475585c9c01c2cce38c8c6e18e7cfa001eb0f4e
3
+ metadata.gz: 239d15fd875ec87669d8db602509802d279c67c2
4
+ data.tar.gz: f0a9342f8e6698c970802049caa0dd2e85e7de93
5
5
  SHA512:
6
- metadata.gz: c07f9db08869a1a74f57442d63f6997d0d0d3da0676c80eb939d9bc09a677c1e2a4cd7f507eaddddcabdbcf4663c953ca7cd688e658a23870dbfaced47351524
7
- data.tar.gz: 260ddbc9e075cab4fd27ab73002bf89b8452b683cc88a609f6be8a5a590dd1bde87bf4b54b8a31acc0f68463d73c90030cb3bc05bd31da98ca6a1444dfda0be0
6
+ metadata.gz: 63095fed8536047dff576cfd78fc2649e721b19cf87154e26207a6508f709a44f2ea176404a4a514715c399cc9d9df105e91c9e82619d3f65f69f9e9d94094cb
7
+ data.tar.gz: aea0d78bd5e95b445a94f956086692814a80c069d0febd90753053522f3fdc49543eaa44fae8666932f81507939db047320e8da5f761e9bc70e774e767941b81
data/README.md CHANGED
@@ -1,38 +1,53 @@
1
- # encoding:utf-8 ;
2
-
3
1
  # extzstd - ruby bindings for Zstd (Zstandard)
4
2
 
5
- This is ruby bindings for compression library
3
+ This is unofficial ruby bindings for compression library
6
4
  [Zstd (https://github.com/Cyan4973/zstd)](https://github.com/Cyan4973/zstd).
7
5
 
8
6
  * package name: extzstd
9
- * author: dearblue (mailto:dearblue@users.osdn.me)
10
- * version: 0.0.2.CONCEPT
7
+ * version: 0.0.3.CONCEPT
11
8
  * software quality: EXPERIMENTAL
12
9
  * license: 2-clause BSD License
13
- * report issue to: https://osdn.jp/projects/rutsubo/ticket/
10
+ * author: dearblue <mailto:dearblue@users.osdn.me>
11
+ * report issue to: <https://osdn.jp/projects/rutsubo/ticket/>
14
12
  * dependency ruby: ruby-2.0+
15
13
  * dependency ruby gems: (none)
16
14
  * dependency library: (none)
17
- * bundled external libraries:
18
- * zstd-0.1.2 (https://github.com/Cyan4973/zstd/tree/zstd-0.1.2)
15
+ * bundled external C library:
16
+ * zstd-0.7.4 <https://github.com/Cyan4973/zstd/tree/v0.7.4>
17
+
18
+ "extzstd" is supported the legacy formats (zstd-0.1 - 0.6).
19
19
 
20
20
 
21
- ## ***WARNING***
21
+ ## Quick API Reference
22
22
 
23
- Zstd data format compatibility is not guaranteed in future versions
24
- (There is a possibility that it becomes impossible to future use).
23
+ * Encode
24
+ * ``Zstd.encode(buf, params = nil, dict = nil) -> encoded string``
25
+ * ``Zstd.encode(outport, params = nil, dict = nil) -> an instance of Zstd::Encoder``
26
+ * ``Zstd.encode(outport, params = nil, dict = nil) { |encoder| ... } -> block returned value``
27
+ * ``Zstd::Encoder#write(buf) -> this instance``
28
+ * ``Zstd::Encoder#close -> nil``
25
29
 
26
- Written in [zstd/README.md](https://github.com/Cyan4973/zstd/blob/zstd-0.1.2/README.md):
30
+ * Decode
31
+ * ``Zstd.decode(zstd_buf, dict = nil) -> decoded string``
32
+ * ``Zstd.decode(inport, dict = nil) -> an intance of Zstd::Decoder``
33
+ * ``Zstd.decode(inport, dict = nil) { |decoder| ... } -> block returned value``
34
+ * ``Zstd::Decoder#read(size = nil, buf = nil) -> buf``
35
+ * ``Zstd::Decoder#close -> nil``
27
36
 
28
- > Zstd has not yet reached "stable" status. Specifically, it doesn't
29
- > guarantee yet that its current compressed format will remain stable
30
- > and supported in future versions.
37
+ * buffered encoder
38
+ * ``Zstd::BufferedEncoder.new(params, dict)`` (``ZBUFF_createCCtx``, ``ZBUFF_compressInit_advanced``)
39
+ * ``Zstd::BufferedEncoder#continue(data, dataoff, dest, maxdest) -> integer as new data offset`` (``ZBUFF_compressContinue``)
40
+ * ``Zstd::BufferedEncoder#flush(dest, maxdest) -> dest`` (``ZBUFF_compressFlush``)
41
+ * ``Zstd::BufferedEncoder#end(dest, maxdest) -> dest`` (``ZBUFF_compressEnd``)
42
+
43
+ * Train dictionary
44
+ * ``Zstd.dict_train_from_buffer(buf, dict_capacity) -> dictionary'd string`` (``ZDICT_trainFromBuffer``)
45
+ * ``Zstd.dict_add_entropy_tables_from_buffer(dict, dict_capacity, sample) -> dict`` (``ZDICT_addEntropyTablesFromBuffer``)
31
46
 
32
47
 
33
48
  ## HOW TO USE
34
49
 
35
- ### basic usage (one pass encode/decode)
50
+ ### basic usage (simply encode/decode)
36
51
 
37
52
  ``` ruby:ruby
38
53
  # First, load library
@@ -41,19 +56,54 @@ require "extzstd"
41
56
  # Prepair data
42
57
  source = "sample data..." * 50
43
58
 
44
- # Directly compression
59
+ # simply compression
45
60
  encdata = Zstd.encode(source)
46
61
  puts "encdata.bytesize=#{encdata.bytesize}"
47
62
 
48
- # Directly decompression
49
- maxdestsize = source.bytesize
50
- decdata = Zstd.decode(encdata, maxdestsize)
63
+ # simply decompression
64
+ decdata = Zstd.decode(encdata)
51
65
  puts "decdata.bytesize=#{decdata.bytesize}"
52
66
 
53
67
  # Comparison source and decoded data
54
68
  p source == decdata # => true
55
69
  ```
56
70
 
57
- ----
71
+ ### Streaming compression (with block)
72
+
73
+ ``` ruby:ruby
74
+ outport = StringIO.new("")
75
+ Zstd.encode(outport) do |encoder|
76
+ encoder.write "abcdefg\n"
77
+ encoder << "hijklmn\n"
78
+ encoder.write "opqrstu\n"
79
+ encoder << "vwxyz\n"
80
+ end
81
+ ```
82
+
83
+ ### Streaming compression (without block and write to file)
58
84
 
59
- [a stub]
85
+ ``` ruby:ruby
86
+ file = File.open("sample.zst", "wb")
87
+ encoder = Zstd.encode(file)
88
+
89
+ encoder.write "abcdefg\n"
90
+ encoder << "hijklmn\n"
91
+ encoder.write "opqrstu\n"
92
+ encoder << "vwxyz\n"
93
+
94
+ encoder.close
95
+ file.close
96
+ ```
97
+
98
+ ### Streaming decompression (with block and read from file)
99
+
100
+ ``` ruby:ruby
101
+ File.open("sample.zst", "rb") do |file|
102
+ Zstd.decode(file) do |decoder|
103
+ p decoder.read(8) # => "abcdefg\n"
104
+ p decoder.read(1) # => "h"
105
+ p decoder.read(2) # => "ij"
106
+ p decoder.read # => "klmn\nopqrstu\nvwxyz\n"
107
+ end
108
+ end
109
+ ```
data/Rakefile CHANGED
@@ -1,11 +1,14 @@
1
1
 
2
+ require "pathname"
2
3
  require "rake/clean"
3
4
 
4
- DOC = FileList["{README,LICENSE,CHANGELOG,Changelog,HISTORY}{,.ja}{,.txt,.rd,.rdoc,.md,.markdown}"] +
5
- FileList["{contrib,ext}/**/{README,LICENSE,CHANGELOG,Changelog,HISTORY}{,.ja}{,.txt,.rd,.rdoc,.md,.markdown}"] +
6
- FileList["ext/**/*.{c,C,cc,cxx,cpp,h,H,hh}"]
7
- #EXT = FileList["ext/**/*.{h,hh,c,cc,cpp,cxx}"] +
8
- # FileList["ext/externals/**/*"]
5
+ docnames = "{README,LICENSE,CHANGELOG,Changelog,HISTORY}"
6
+ doctypes = "{,.txt,.rd,.rdoc,.md,.markdown}"
7
+ cexttypes = "{c,C,cc,cxx,cpp,h,H,hh}"
8
+
9
+ DOC = FileList["#{docnames}{,.ja}#{doctypes}"] +
10
+ FileList["{contrib,ext}/**/#{docnames}{,.ja}#{doctypes}"] +
11
+ FileList["ext/**/*.#{cexttypes}"]
9
12
  EXT = FileList["ext/**/*"]
10
13
  BIN = FileList["bin/*"]
11
14
  LIB = FileList["lib/**/*.rb"]
@@ -15,18 +18,23 @@ EXAMPLE = FileList["examples/**/*"]
15
18
  GEMSTUB_SRC = "gemstub.rb"
16
19
  RAKEFILE = [File.basename(__FILE__), GEMSTUB_SRC]
17
20
  EXTRA = []
21
+ EXTCONF = FileList["ext/**/extconf.rb"]
22
+ EXTCONF.reject! { |n| !File.file?(n) }
23
+ EXTMAP = {}
18
24
 
19
25
  load GEMSTUB_SRC
20
26
 
21
- EXTCONF = FileList["ext/extconf.rb"]
22
- EXTCONF.reject! { |n| !File.file?(n) }
27
+ EXTMAP.dup.each_pair do |dir, name|
28
+ EXTMAP[Pathname.new(dir).cleanpath.to_s] = Pathname.new(name).cleanpath.to_s
29
+ end
30
+
23
31
  GEMSTUB.extensions += EXTCONF
24
32
  GEMSTUB.executables += FileList["bin/*"].map { |n| File.basename n }
25
33
  GEMSTUB.executables.sort!
26
34
 
27
35
  PACKAGENAME = "#{GEMSTUB.name}-#{GEMSTUB.version}"
28
36
  GEMFILE = "#{PACKAGENAME}.gem"
29
- GEMSPEC = "#{GEMSTUB.name}.gemspec"
37
+ GEMSPEC = "#{PACKAGENAME}.gemspec"
30
38
 
31
39
  GEMSTUB.files += DOC + EXT + EXTCONF + BIN + LIB + SPEC + TEST + EXAMPLE + RAKEFILE + EXTRA
32
40
  GEMSTUB.files.sort!
@@ -37,11 +45,27 @@ end
37
45
  GEMSTUB.extra_rdoc_files += DOC + LIB + EXT.reject { |n| n.include?("/externals/") || !%w(.h .hh .c .cc .cpp .cxx).include?(File.extname(n)) }
38
46
  GEMSTUB.extra_rdoc_files.sort!
39
47
 
40
- CLEAN << GEMSPEC
48
+ GEMSTUB_TRYOUT = GEMSTUB.dup
49
+ GEMSTUB_TRYOUT.version = "#{GEMSTUB.version}#{Time.now.strftime(".TRYOUT.%Y%m%d.%H%M%S")}"
50
+ PACKAGENAME_TRYOUT = "#{GEMSTUB.name}-#{GEMSTUB_TRYOUT.version}"
51
+ GEMFILE_TRYOUT = "#{PACKAGENAME_TRYOUT}.gem"
52
+ GEMSPEC_TRYOUT = "#{PACKAGENAME_TRYOUT}.gemspec"
53
+
54
+ CLEAN << GEMSPEC << GEMSPEC_TRYOUT
41
55
  CLOBBER << GEMFILE
42
56
 
43
- task :default => :all
57
+ task :default => :tryout do
58
+ $stderr.puts <<-EOS
59
+ #{__FILE__}:#{__LINE__}:
60
+ \ttype ``rake release'' to build release package.
61
+ EOS
62
+ end
63
+
64
+ desc "build tryout package"
65
+ task :tryout
44
66
 
67
+ desc "build release package"
68
+ task :release => :all
45
69
 
46
70
  unless EXTCONF.empty?
47
71
  RUBYSET ||= (ENV["RUBYSET"] || "").split(",")
@@ -54,24 +78,43 @@ unless EXTCONF.empty?
54
78
  | variable for set ruby interpreters by comma separated.
55
79
  |
56
80
  | e.g.) $ rake RUBYSET=ruby
57
- | or) $ rake RUBYSET=ruby20,ruby21,ruby22
81
+ | or) $ rake RUBYSET=ruby21,ruby22,ruby23
58
82
  |
59
83
  EOS
60
84
  else
61
- platforms = RUBYSET.map { |ruby| `#{ruby} --disable gems -rrbconfig -e "puts RbConfig::CONFIG['arch']"`.chomp }
85
+ platforms = RUBYSET.map { |ruby| `#{ruby} --disable-gems -e "puts RUBY_PLATFORM"`.chomp }
62
86
  platforms1 = platforms.uniq
63
87
  unless platforms1.size == 1 && !platforms1[0].empty?
64
- raise "different platforms (#{Hash[*RUBYSET.zip(platforms).flatten].inspect})"
88
+ abort <<-EOS
89
+ #{__FILE__}:#{__LINE__}: different platforms:
90
+ #{RUBYSET.zip(platforms).map { |ruby, platform| "%24s => %s" % [ruby, platform] }.join("\n")}
91
+ ABORTED.
92
+ EOS
65
93
  end
66
94
  PLATFORM = platforms1[0]
67
95
 
68
96
  RUBY_VERSIONS = RUBYSET.map do |ruby|
69
- ver = `#{ruby} --disable gem -rrbconfig -e "puts RbConfig::CONFIG['ruby_version']"`.slice(/\d+\.\d+/)
97
+ ver = `#{ruby} --disable-gems -e "puts RUBY_VERSION"`.slice(/\d+\.\d+/)
70
98
  raise "failed ruby checking - ``#{ruby}''" unless $?.success?
71
99
  [ver, ruby]
72
100
  end
73
- SOFILES_SET = RUBY_VERSIONS.map { |(ver, ruby)| ["lib/#{ver}/#{GEMSTUB.name}.so", ruby] }
74
- SOFILES = SOFILES_SET.map { |(lib, ruby)| lib }
101
+
102
+ SOFILES_SET = RUBY_VERSIONS.map { |(ver, ruby)|
103
+ EXTCONF.map { |extconf|
104
+ extdir = Pathname.new(extconf).cleanpath.dirname.to_s
105
+ case
106
+ when soname = EXTMAP[extdir.sub(/^ext\//i, "")]
107
+ soname = soname.sub(/\.so$/i, "")
108
+ when extdir == "ext" || extdir == "."
109
+ soname = GEMSTUB.name
110
+ else
111
+ soname = File.basename(extdir)
112
+ end
113
+
114
+ [ruby, File.join("lib", "#{soname.sub(/(?<=\/)|^(?!.*\/)/, "#{ver}/")}.so"), extconf]
115
+ }
116
+ }.flatten(1)
117
+ SOFILES = SOFILES_SET.map { |(ruby, sopath, extconf)| sopath }
75
118
 
76
119
  GEMSTUB_NATIVE = GEMSTUB.dup
77
120
  GEMSTUB_NATIVE.files += SOFILES
@@ -99,7 +142,7 @@ unless EXTCONF.empty?
99
142
  desc "build c-extension libraries"
100
143
  task "sofiles" => SOFILES
101
144
 
102
- SOFILES_SET.each do |(soname, ruby)|
145
+ SOFILES_SET.each do |(ruby, soname, extconf)|
103
146
  sodir = File.dirname(soname)
104
147
  makefile = File.join(sodir, "Makefile")
105
148
 
@@ -109,9 +152,10 @@ unless EXTCONF.empty?
109
152
  directory sodir
110
153
 
111
154
  desc "generate Makefile for binary extension library"
112
- file makefile => [sodir] + EXTCONF do
155
+ file makefile => [sodir, extconf] do
156
+ rel_extconf = Pathname.new(extconf).relative_path_from(Pathname.new(sodir)).to_s
113
157
  cd sodir do
114
- sh "#{ruby} ../../#{EXTCONF[0]} \"--ruby=#{ruby}\""
158
+ sh *%W"#{ruby} #{rel_extconf} --ruby=#{ruby} #{ENV["EXTCONF"]}"
115
159
  end
116
160
  end
117
161
 
@@ -127,6 +171,7 @@ end
127
171
 
128
172
 
129
173
  task :all => GEMFILE
174
+ task :tryout => GEMFILE_TRYOUT
130
175
 
131
176
  desc "generate local rdoc"
132
177
  task :rdoc => DOC + LIB do
@@ -153,6 +198,15 @@ file GEMFILE => DOC + EXT + EXTCONF + BIN + LIB + SPEC + TEST + EXAMPLE + RAKEFI
153
198
  sh "gem build #{GEMSPEC}"
154
199
  end
155
200
 
201
+ file GEMFILE_TRYOUT => DOC + EXT + EXTCONF + BIN + LIB + SPEC + TEST + EXAMPLE + RAKEFILE + [GEMSPEC_TRYOUT] do
202
+ #file GEMFILE_TRYOUT do
203
+ sh "gem build #{GEMSPEC_TRYOUT}"
204
+ end
205
+
156
206
  file GEMSPEC => RAKEFILE do
157
207
  File.write(GEMSPEC, GEMSTUB.to_ruby, mode: "wb")
158
208
  end
209
+
210
+ file GEMSPEC_TRYOUT => RAKEFILE do
211
+ File.write(GEMSPEC_TRYOUT, GEMSTUB_TRYOUT.to_ruby, mode: "wb")
212
+ end
@@ -0,0 +1,68 @@
1
+ zstd - library files
2
+ ================================
3
+
4
+ The __lib__ directory contains several files, but depending on target use case, some of them may not be necessary.
5
+
6
+ #### Minimal library files
7
+
8
+ To build the zstd library the following files are required:
9
+
10
+ - [common/bitstream.h](common/bitstream.h)
11
+ - [common/error_private.h](common/error_private.h)
12
+ - [common/error_public.h](common/error_public.h)
13
+ - common/fse.h
14
+ - common/fse_decompress.c
15
+ - common/huf.h
16
+ - [common/mem.h](common/mem.h)
17
+ - [common/zstd.h]
18
+ - common/zstd_internal.h
19
+ - compress/fse_compress.c
20
+ - compress/huf_compress.c
21
+ - compress/zstd_compress.c
22
+ - compress/zstd_opt.h
23
+ - decompress/huf_decompress.c
24
+ - decompress/zstd_decompress.c
25
+
26
+ Stable API is exposed in [common/zstd.h].
27
+ Advanced and experimental API can be enabled by defining `ZSTD_STATIC_LINKING_ONLY`.
28
+ Never use them with a dynamic library, as their definition may change in future versions.
29
+
30
+ [common/zstd.h]: common/zstd.h
31
+
32
+
33
+ #### Separate compressor and decompressor
34
+
35
+ To build a separate zstd compressor all files from `common/` and `compressor/` directories are required.
36
+ In a similar way to build a separate zstd decompressor all files from `common/` and `decompressor/` directories are needed.
37
+
38
+
39
+ #### Buffered streaming
40
+
41
+ This complementary API makes streaming integration easier.
42
+ It is used by `zstd` command line utility, and [7zip plugin](http://mcmilk.de/projects/7-Zip-ZStd) :
43
+
44
+ - common/zbuff.h
45
+ - compress/zbuff_compress.c
46
+ - decompress/zbuff_decompress.c
47
+
48
+
49
+ #### Dictionary builder
50
+
51
+ In order to create dictionaries from some training sets,
52
+ it's needed to include all files from [dictBuilder directory](dictBuilder/)
53
+
54
+
55
+ #### Legacy support
56
+
57
+ Zstandard can decode previous formats, starting from v0.1.
58
+ Support for these format is provided in [folder legacy](legacy/).
59
+ It's also required to compile the library with `ZSTD_LEGACY_SUPPORT = 1`.
60
+
61
+
62
+ #### Miscellaneous
63
+
64
+ The other files are not source code. There are :
65
+
66
+ - LICENSE : contains the BSD license text
67
+ - Makefile : script to compile or install zstd library (static or dynamic)
68
+ - libzstd.pc.in : for pkg-config (make install)
@@ -0,0 +1,414 @@
1
+ /* ******************************************************************
2
+ bitstream
3
+ Part of FSE library
4
+ header file (to include)
5
+ Copyright (C) 2013-2016, Yann Collet.
6
+
7
+ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
8
+
9
+ Redistribution and use in source and binary forms, with or without
10
+ modification, are permitted provided that the following conditions are
11
+ met:
12
+
13
+ * Redistributions of source code must retain the above copyright
14
+ notice, this list of conditions and the following disclaimer.
15
+ * Redistributions in binary form must reproduce the above
16
+ copyright notice, this list of conditions and the following disclaimer
17
+ in the documentation and/or other materials provided with the
18
+ distribution.
19
+
20
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31
+
32
+ You can contact the author at :
33
+ - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
34
+ ****************************************************************** */
35
+ #ifndef BITSTREAM_H_MODULE
36
+ #define BITSTREAM_H_MODULE
37
+
38
+ #if defined (__cplusplus)
39
+ extern "C" {
40
+ #endif
41
+
42
+
43
+ /*
44
+ * This API consists of small unitary functions, which must be inlined for best performance.
45
+ * Since link-time-optimization is not available for all compilers,
46
+ * these functions are defined into a .h to be included.
47
+ */
48
+
49
+ /*-****************************************
50
+ * Dependencies
51
+ ******************************************/
52
+ #include "mem.h" /* unaligned access routines */
53
+ #include "error_private.h" /* error codes and messages */
54
+
55
+
56
+ /*=========================================
57
+ * Target specific
58
+ =========================================*/
59
+ #if defined(__BMI__) && defined(__GNUC__)
60
+ # include <immintrin.h> /* support for bextr (experimental) */
61
+ #endif
62
+
63
+
64
+ /*-******************************************
65
+ * bitStream encoding API (write forward)
66
+ ********************************************/
67
+ /* bitStream can mix input from multiple sources.
68
+ * A critical property of these streams is that they encode and decode in **reverse** direction.
69
+ * So the first bit sequence you add will be the last to be read, like a LIFO stack.
70
+ */
71
+ typedef struct
72
+ {
73
+ size_t bitContainer;
74
+ int bitPos;
75
+ char* startPtr;
76
+ char* ptr;
77
+ char* endPtr;
78
+ } BIT_CStream_t;
79
+
80
+ MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* dstBuffer, size_t dstCapacity);
81
+ MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits);
82
+ MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC);
83
+ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC);
84
+
85
+ /* Start with initCStream, providing the size of buffer to write into.
86
+ * bitStream will never write outside of this buffer.
87
+ * `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code.
88
+ *
89
+ * bits are first added to a local register.
90
+ * Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems.
91
+ * Writing data into memory is an explicit operation, performed by the flushBits function.
92
+ * Hence keep track how many bits are potentially stored into local register to avoid register overflow.
93
+ * After a flushBits, a maximum of 7 bits might still be stored into local register.
94
+ *
95
+ * Avoid storing elements of more than 24 bits if you want compatibility with 32-bits bitstream readers.
96
+ *
97
+ * Last operation is to close the bitStream.
98
+ * The function returns the final size of CStream in bytes.
99
+ * If data couldn't fit into `dstBuffer`, it will return a 0 ( == not storable)
100
+ */
101
+
102
+
103
+ /*-********************************************
104
+ * bitStream decoding API (read backward)
105
+ **********************************************/
106
+ typedef struct
107
+ {
108
+ size_t bitContainer;
109
+ unsigned bitsConsumed;
110
+ const char* ptr;
111
+ const char* start;
112
+ } BIT_DStream_t;
113
+
114
+ typedef enum { BIT_DStream_unfinished = 0,
115
+ BIT_DStream_endOfBuffer = 1,
116
+ BIT_DStream_completed = 2,
117
+ BIT_DStream_overflow = 3 } BIT_DStream_status; /* result of BIT_reloadDStream() */
118
+ /* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */
119
+
120
+ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize);
121
+ MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits);
122
+ MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD);
123
+ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
124
+
125
+
126
+ /* Start by invoking BIT_initDStream().
127
+ * A chunk of the bitStream is then stored into a local register.
128
+ * Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
129
+ * You can then retrieve bitFields stored into the local register, **in reverse order**.
130
+ * Local register is explicitly reloaded from memory by the BIT_reloadDStream() method.
131
+ * A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished.
132
+ * Otherwise, it can be less than that, so proceed accordingly.
133
+ * Checking if DStream has reached its end can be performed with BIT_endOfDStream().
134
+ */
135
+
136
+
137
+ /*-****************************************
138
+ * unsafe API
139
+ ******************************************/
140
+ MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits);
141
+ /* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */
142
+
143
+ MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC);
144
+ /* unsafe version; does not check buffer overflow */
145
+
146
+ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
147
+ /* faster, but works only if nbBits >= 1 */
148
+
149
+
150
+
151
+ /*-**************************************************************
152
+ * Internal functions
153
+ ****************************************************************/
154
+ MEM_STATIC unsigned BIT_highbit32 (register U32 val)
155
+ {
156
+ # if defined(_MSC_VER) /* Visual */
157
+ unsigned long r=0;
158
+ _BitScanReverse ( &r, val );
159
+ return (unsigned) r;
160
+ # elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
161
+ return 31 - __builtin_clz (val);
162
+ # else /* Software version */
163
+ static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
164
+ U32 v = val;
165
+ v |= v >> 1;
166
+ v |= v >> 2;
167
+ v |= v >> 4;
168
+ v |= v >> 8;
169
+ v |= v >> 16;
170
+ return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27];
171
+ # endif
172
+ }
173
+
174
+ /*===== Local Constants =====*/
175
+ static const unsigned BIT_mask[] = { 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF }; /* up to 26 bits */
176
+
177
+
178
+ /*-**************************************************************
179
+ * bitStream encoding
180
+ ****************************************************************/
181
+ /*! BIT_initCStream() :
182
+ * `dstCapacity` must be > sizeof(void*)
183
+ * @return : 0 if success,
184
+ otherwise an error code (can be tested using ERR_isError() ) */
185
+ MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* startPtr, size_t dstCapacity)
186
+ {
187
+ bitC->bitContainer = 0;
188
+ bitC->bitPos = 0;
189
+ bitC->startPtr = (char*)startPtr;
190
+ bitC->ptr = bitC->startPtr;
191
+ bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->ptr);
192
+ if (dstCapacity <= sizeof(bitC->ptr)) return ERROR(dstSize_tooSmall);
193
+ return 0;
194
+ }
195
+
196
+ /*! BIT_addBits() :
197
+ can add up to 26 bits into `bitC`.
198
+ Does not check for register overflow ! */
199
+ MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits)
200
+ {
201
+ bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos;
202
+ bitC->bitPos += nbBits;
203
+ }
204
+
205
+ /*! BIT_addBitsFast() :
206
+ * works only if `value` is _clean_, meaning all high bits above nbBits are 0 */
207
+ MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits)
208
+ {
209
+ bitC->bitContainer |= value << bitC->bitPos;
210
+ bitC->bitPos += nbBits;
211
+ }
212
+
213
+ /*! BIT_flushBitsFast() :
214
+ * unsafe version; does not check buffer overflow */
215
+ MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC)
216
+ {
217
+ size_t const nbBytes = bitC->bitPos >> 3;
218
+ MEM_writeLEST(bitC->ptr, bitC->bitContainer);
219
+ bitC->ptr += nbBytes;
220
+ bitC->bitPos &= 7;
221
+ bitC->bitContainer >>= nbBytes*8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */
222
+ }
223
+
224
+ /*! BIT_flushBits() :
225
+ * safe version; check for buffer overflow, and prevents it.
226
+ * note : does not signal buffer overflow. This will be revealed later on using BIT_closeCStream() */
227
+ MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC)
228
+ {
229
+ size_t const nbBytes = bitC->bitPos >> 3;
230
+ MEM_writeLEST(bitC->ptr, bitC->bitContainer);
231
+ bitC->ptr += nbBytes;
232
+ if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr;
233
+ bitC->bitPos &= 7;
234
+ bitC->bitContainer >>= nbBytes*8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */
235
+ }
236
+
237
+ /*! BIT_closeCStream() :
238
+ * @return : size of CStream, in bytes,
239
+ or 0 if it could not fit into dstBuffer */
240
+ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC)
241
+ {
242
+ BIT_addBitsFast(bitC, 1, 1); /* endMark */
243
+ BIT_flushBits(bitC);
244
+
245
+ if (bitC->ptr >= bitC->endPtr) return 0; /* doesn't fit within authorized budget : cancel */
246
+
247
+ return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0);
248
+ }
249
+
250
+
251
+ /*-********************************************************
252
+ * bitStream decoding
253
+ **********************************************************/
254
+ /*! BIT_initDStream() :
255
+ * Initialize a BIT_DStream_t.
256
+ * `bitD` : a pointer to an already allocated BIT_DStream_t structure.
257
+ * `srcSize` must be the *exact* size of the bitStream, in bytes.
258
+ * @return : size of stream (== srcSize) or an errorCode if a problem is detected
259
+ */
260
+ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize)
261
+ {
262
+ if (srcSize < 1) { memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); }
263
+
264
+ if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */
265
+ bitD->start = (const char*)srcBuffer;
266
+ bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer);
267
+ bitD->bitContainer = MEM_readLEST(bitD->ptr);
268
+ { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
269
+ bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0;
270
+ if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ }
271
+ } else {
272
+ bitD->start = (const char*)srcBuffer;
273
+ bitD->ptr = bitD->start;
274
+ bitD->bitContainer = *(const BYTE*)(bitD->start);
275
+ switch(srcSize)
276
+ {
277
+ case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16);
278
+ case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24);
279
+ case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32);
280
+ case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24;
281
+ case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16;
282
+ case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8;
283
+ default:;
284
+ }
285
+ { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
286
+ bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0;
287
+ if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ }
288
+ bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8;
289
+ }
290
+
291
+ return srcSize;
292
+ }
293
+
294
+ MEM_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start)
295
+ {
296
+ return bitContainer >> start;
297
+ }
298
+
299
+ MEM_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits)
300
+ {
301
+ #if defined(__BMI__) && defined(__GNUC__) /* experimental */
302
+ # if defined(__x86_64__)
303
+ if (sizeof(bitContainer)==8)
304
+ return _bextr_u64(bitContainer, start, nbBits);
305
+ else
306
+ # endif
307
+ return _bextr_u32(bitContainer, start, nbBits);
308
+ #else
309
+ return (bitContainer >> start) & BIT_mask[nbBits];
310
+ #endif
311
+ }
312
+
313
+ MEM_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits)
314
+ {
315
+ return bitContainer & BIT_mask[nbBits];
316
+ }
317
+
318
+ /*! BIT_lookBits() :
319
+ * Provides next n bits from local register.
320
+ * local register is not modified.
321
+ * On 32-bits, maxNbBits==24.
322
+ * On 64-bits, maxNbBits==56.
323
+ * @return : value extracted
324
+ */
325
+ MEM_STATIC size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits)
326
+ {
327
+ #if defined(__BMI__) && defined(__GNUC__) /* experimental; fails if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8 */
328
+ return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits);
329
+ #else
330
+ U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1;
331
+ return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask);
332
+ #endif
333
+ }
334
+
335
+ /*! BIT_lookBitsFast() :
336
+ * unsafe version; only works only if nbBits >= 1 */
337
+ MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits)
338
+ {
339
+ U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1;
340
+ return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask+1)-nbBits) & bitMask);
341
+ }
342
+
343
+ MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
344
+ {
345
+ bitD->bitsConsumed += nbBits;
346
+ }
347
+
348
+ /*! BIT_readBits() :
349
+ * Read (consume) next n bits from local register and update.
350
+ * Pay attention to not read more than nbBits contained into local register.
351
+ * @return : extracted value.
352
+ */
353
+ MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
354
+ {
355
+ size_t const value = BIT_lookBits(bitD, nbBits);
356
+ BIT_skipBits(bitD, nbBits);
357
+ return value;
358
+ }
359
+
360
+ /*! BIT_readBitsFast() :
361
+ * unsafe version; only works only if nbBits >= 1 */
362
+ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
363
+ {
364
+ size_t const value = BIT_lookBitsFast(bitD, nbBits);
365
+ BIT_skipBits(bitD, nbBits);
366
+ return value;
367
+ }
368
+
369
+ /*! BIT_reloadDStream() :
370
+ * Refill `BIT_DStream_t` from src buffer previously defined (see BIT_initDStream() ).
371
+ * This function is safe, it guarantees it will not read beyond src buffer.
372
+ * @return : status of `BIT_DStream_t` internal register.
373
+ if status == unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */
374
+ MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
375
+ {
376
+ if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should not happen => corruption detected */
377
+ return BIT_DStream_overflow;
378
+
379
+ if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) {
380
+ bitD->ptr -= bitD->bitsConsumed >> 3;
381
+ bitD->bitsConsumed &= 7;
382
+ bitD->bitContainer = MEM_readLEST(bitD->ptr);
383
+ return BIT_DStream_unfinished;
384
+ }
385
+ if (bitD->ptr == bitD->start) {
386
+ if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer;
387
+ return BIT_DStream_completed;
388
+ }
389
+ { U32 nbBytes = bitD->bitsConsumed >> 3;
390
+ BIT_DStream_status result = BIT_DStream_unfinished;
391
+ if (bitD->ptr - nbBytes < bitD->start) {
392
+ nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */
393
+ result = BIT_DStream_endOfBuffer;
394
+ }
395
+ bitD->ptr -= nbBytes;
396
+ bitD->bitsConsumed -= nbBytes*8;
397
+ bitD->bitContainer = MEM_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD) */
398
+ return result;
399
+ }
400
+ }
401
+
402
+ /*! BIT_endOfDStream() :
403
+ * @return Tells if DStream has exactly reached its end (all bits consumed).
404
+ */
405
+ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)
406
+ {
407
+ return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8));
408
+ }
409
+
410
+ #if defined (__cplusplus)
411
+ }
412
+ #endif
413
+
414
+ #endif /* BITSTREAM_H_MODULE */