omnizip 0.3.13 → 0.3.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/omnizip/algorithm.rb +36 -1
- data/lib/omnizip/algorithms/bzip2.rb +1 -3
- data/lib/omnizip/algorithms/deflate.rb +1 -3
- data/lib/omnizip/algorithms/deflate64.rb +2 -2
- data/lib/omnizip/algorithms/lzma/optimal_encoder.rb +7 -8
- data/lib/omnizip/algorithms/lzma.rb +1 -1
- data/lib/omnizip/algorithms/lzma2/lzma2_chunk.rb +5 -1
- data/lib/omnizip/algorithms/lzma2.rb +4 -12
- data/lib/omnizip/algorithms/zstandard/constants.rb +38 -25
- data/lib/omnizip/algorithms/zstandard/decoder.rb +101 -136
- data/lib/omnizip/algorithms/zstandard/encoder.rb +81 -138
- data/lib/omnizip/algorithms/zstandard/frame/block.rb +17 -0
- data/lib/omnizip/algorithms/zstandard/frame/header.rb +42 -29
- data/lib/omnizip/algorithms/zstandard/fse/bitstream.rb +114 -117
- data/lib/omnizip/algorithms/zstandard/fse/encoder.rb +434 -222
- data/lib/omnizip/algorithms/zstandard/fse/interleaved.rb +92 -0
- data/lib/omnizip/algorithms/zstandard/fse/table.rb +98 -176
- data/lib/omnizip/algorithms/zstandard/fse/table_description.rb +203 -0
- data/lib/omnizip/algorithms/zstandard/fse.rb +21 -2
- data/lib/omnizip/algorithms/zstandard/huffman.rb +195 -188
- data/lib/omnizip/algorithms/zstandard/huffman_encoder.rb +232 -255
- data/lib/omnizip/algorithms/zstandard/literals.rb +170 -103
- data/lib/omnizip/algorithms/zstandard/literals_encoder.rb +47 -199
- data/lib/omnizip/algorithms/zstandard/sequences.rb +260 -252
- data/lib/omnizip/algorithms/zstandard/xxhash.rb +159 -0
- data/lib/omnizip/algorithms/zstandard.rb +20 -24
- data/lib/omnizip/algorithms.rb +14 -1
- data/lib/omnizip/buffer.rb +1 -2
- data/lib/omnizip/commands/archive_list_command.rb +9 -7
- data/lib/omnizip/convenience.rb +1 -2
- data/lib/omnizip/entry.rb +44 -0
- data/lib/omnizip/extraction/selective_extractor.rb +2 -10
- data/lib/omnizip/filter_pipeline.rb +1 -1
- data/lib/omnizip/filter_registry.rb +18 -16
- data/lib/omnizip/filters/bcj2.rb +1 -1
- data/lib/omnizip/filters/bcj_arm.rb +1 -1
- data/lib/omnizip/filters/bcj_arm64.rb +1 -1
- data/lib/omnizip/filters/bcj_ia64.rb +1 -1
- data/lib/omnizip/filters/bcj_ppc.rb +1 -1
- data/lib/omnizip/filters/bcj_sparc.rb +1 -1
- data/lib/omnizip/filters/bcj_x86.rb +1 -1
- data/lib/omnizip/filters.rb +0 -2
- data/lib/omnizip/formats/bzip2_file.rb +0 -7
- data/lib/omnizip/formats/cpio/entry.rb +6 -0
- data/lib/omnizip/formats/cpio.rb +0 -6
- data/lib/omnizip/formats/gzip.rb +0 -7
- data/lib/omnizip/formats/iso/directory_record.rb +7 -0
- data/lib/omnizip/formats/iso.rb +0 -5
- data/lib/omnizip/formats/lzip.rb +0 -7
- data/lib/omnizip/formats/lzma_alone.rb +0 -6
- data/lib/omnizip/formats/msi/entry.rb +6 -0
- data/lib/omnizip/formats/msi.rb +0 -9
- data/lib/omnizip/formats/ole/dirent.rb +6 -0
- data/lib/omnizip/formats/ole.rb +0 -9
- data/lib/omnizip/formats/rar.rb +0 -6
- data/lib/omnizip/formats/rar3/reader.rb +7 -0
- data/lib/omnizip/formats/rar5/reader.rb +7 -0
- data/lib/omnizip/formats/rpm/entry.rb +6 -0
- data/lib/omnizip/formats/seven_zip/models/file_entry.rb +7 -0
- data/lib/omnizip/formats/seven_zip.rb +0 -6
- data/lib/omnizip/formats/tar/entry.rb +6 -0
- data/lib/omnizip/formats/tar.rb +2 -10
- data/lib/omnizip/formats/xar/entry.rb +6 -0
- data/lib/omnizip/formats/xar.rb +0 -6
- data/lib/omnizip/formats/zip.rb +0 -6
- data/lib/omnizip/implementations/seven_zip/lzma2/encoder.rb +33 -31
- data/lib/omnizip/implementations/xz_utils/lzma2/encoder.rb +177 -163
- data/lib/omnizip/implementations.rb +18 -4
- data/lib/omnizip/io/source.rb +3 -0
- data/lib/omnizip/metadata/entry_metadata.rb +7 -0
- data/lib/omnizip/parallel/engine.rb +48 -0
- data/lib/omnizip/parallel/parallel_compressor.rb +5 -16
- data/lib/omnizip/parallel/parallel_extractor.rb +5 -16
- data/lib/omnizip/parallel.rb +1 -0
- data/lib/omnizip/profile/profile_registry.rb +1 -2
- data/lib/omnizip/registry.rb +0 -1
- data/lib/omnizip/rubyzip_compat.rb +0 -1
- data/lib/omnizip/temp/temp_file.rb +1 -2
- data/lib/omnizip/version.rb +1 -1
- data/lib/omnizip/zip/entry.rb +7 -0
- data/lib/omnizip/zip/file.rb +4 -8
- data/lib/omnizip.rb +1 -1
- metadata +7 -6
- data/lib/omnizip/filters/filter_base.rb +0 -6
- data/lib/omnizip/filters/registration.rb +0 -22
- data/lib/omnizip/filters/registry.rb +0 -111
- data/lib/omnizip/format_registry.rb +0 -100
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 978a7f51735e4fd90dbb0432b933ccdd17009d74c682db94f6ee6730fe1f893a
|
|
4
|
+
data.tar.gz: c02f967a72d720edbbc204bd6b36dc60a4c4f773aed3e308800880351a32c218
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 6664a917d2c82bca2779e62a082e1462c1dd8c36f3f4777a47a871feafd4d3bb25e7ae4eb492623a823aed8755777fce93374075a502b0a8beabe2becf9aed05
|
|
7
|
+
data.tar.gz: ccf0ff7279dc03c4c5aec2fd6a679419c3fa275966e3b7f076e3bdd4e5890e2141664f6137df46681d29738dbf7fb25ef95e1ec3e450c0e3cfc7509dd9f759d7
|
data/lib/omnizip/algorithm.rb
CHANGED
|
@@ -23,6 +23,8 @@ module Omnizip
|
|
|
23
23
|
# the required methods. Algorithms are automatically registered with the
|
|
24
24
|
# AlgorithmRegistry when defined.
|
|
25
25
|
class Algorithm
|
|
26
|
+
require "stringio"
|
|
27
|
+
|
|
26
28
|
attr_reader :options, :filter
|
|
27
29
|
|
|
28
30
|
# Initialize algorithm with options.
|
|
@@ -38,7 +40,7 @@ module Omnizip
|
|
|
38
40
|
# The filter will be applied before compression and reversed after
|
|
39
41
|
# decompression. Returns self for method chaining.
|
|
40
42
|
#
|
|
41
|
-
# @param filter [
|
|
43
|
+
# @param filter [Omnizip::Filter, FilterPipeline] Filter or
|
|
42
44
|
# pipeline to use
|
|
43
45
|
# @return [self] For method chaining
|
|
44
46
|
def with_filter(filter)
|
|
@@ -82,6 +84,39 @@ module Omnizip
|
|
|
82
84
|
"#{self} must implement .metadata"
|
|
83
85
|
end
|
|
84
86
|
|
|
87
|
+
# Class-level convenience: compress `data` and return the
|
|
88
|
+
# compressed bytes. Carries the algorithm's streaming contract
|
|
89
|
+
# over a single StringIO so callers with the whole payload in
|
|
90
|
+
# memory (parallel compressors, small files) don't need to
|
|
91
|
+
# instantiate and wire streams themselves.
|
|
92
|
+
#
|
|
93
|
+
# @param data [String, IO] Input to compress
|
|
94
|
+
# @param options [Hash] Forwarded to the algorithm
|
|
95
|
+
# @return [String] Compressed bytes
|
|
96
|
+
def compress(data, **options)
|
|
97
|
+
instance = new(options)
|
|
98
|
+
input = data.is_a?(::IO) ? data : ::StringIO.new(data.to_s.b)
|
|
99
|
+
output = ::StringIO.new
|
|
100
|
+
output.set_encoding(Encoding::BINARY)
|
|
101
|
+
instance.compress(input, output, options)
|
|
102
|
+
output.string
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# Class-level convenience: decompress `data` and return the
|
|
106
|
+
# uncompressed bytes. Mirror of +.compress+.
|
|
107
|
+
#
|
|
108
|
+
# @param data [String, IO] Input to decompress
|
|
109
|
+
# @param options [Hash] Forwarded to the algorithm
|
|
110
|
+
# @return [String] Decompressed bytes
|
|
111
|
+
def decompress(data, **options)
|
|
112
|
+
instance = new(options)
|
|
113
|
+
input = data.is_a?(::IO) ? data : ::StringIO.new(data.to_s.b)
|
|
114
|
+
output = ::StringIO.new
|
|
115
|
+
output.set_encoding(Encoding::BINARY)
|
|
116
|
+
instance.decompress(input, output, options)
|
|
117
|
+
output.string
|
|
118
|
+
end
|
|
119
|
+
|
|
85
120
|
# Automatically register algorithm when inherited.
|
|
86
121
|
#
|
|
87
122
|
# This hook is called whenever a class inherits from Algorithm,
|
|
@@ -82,9 +82,7 @@ module Omnizip
|
|
|
82
82
|
# @param options [Models::CompressionOptions] Decompression options
|
|
83
83
|
# @return [void]
|
|
84
84
|
def decompress(input_stream, output_stream, _options = nil)
|
|
85
|
-
|
|
86
|
-
output_stream.set_encoding(Encoding::BINARY)
|
|
87
|
-
end
|
|
85
|
+
output_stream.set_encoding(Encoding::BINARY)
|
|
88
86
|
decoder = Decoder.new(input_stream)
|
|
89
87
|
decompressed = decoder.decode_stream
|
|
90
88
|
output_stream.write(decompressed)
|
|
@@ -77,9 +77,7 @@ module Omnizip
|
|
|
77
77
|
# @param options [Models::CompressionOptions] Decompression options
|
|
78
78
|
# @return [void]
|
|
79
79
|
def decompress(input_stream, output_stream, _options = nil)
|
|
80
|
-
|
|
81
|
-
output_stream.set_encoding(Encoding::BINARY)
|
|
82
|
-
end
|
|
80
|
+
output_stream.set_encoding(Encoding::BINARY)
|
|
83
81
|
decoder = Decoder.new(input_stream)
|
|
84
82
|
decompressed = decoder.decode_stream
|
|
85
83
|
output_stream.write(decompressed)
|
|
@@ -73,8 +73,8 @@ module Omnizip
|
|
|
73
73
|
return if compressed.nil? || compressed.empty?
|
|
74
74
|
|
|
75
75
|
# Set output to binary mode if it's a StringIO
|
|
76
|
-
output.set_encoding(Encoding::BINARY)
|
|
77
|
-
output.binmode
|
|
76
|
+
output.set_encoding(Encoding::BINARY)
|
|
77
|
+
output.binmode
|
|
78
78
|
|
|
79
79
|
# Use Zlib::Inflate with maximum window size
|
|
80
80
|
inflater = Zlib::Inflate.new(Zlib::MAX_WBITS)
|
|
@@ -100,13 +100,14 @@ module Omnizip
|
|
|
100
100
|
|
|
101
101
|
# We didn't find a long enough repeated match.
|
|
102
102
|
# Encode it as a normal match if the match length is at least nice_len
|
|
103
|
-
# AND the match distance
|
|
103
|
+
# AND the match distance stays within the data decoded so far.
|
|
104
|
+
# A match may reference anything before `position`: bytes already
|
|
105
|
+
# committed to the dictionary *and* bytes decoded earlier within
|
|
106
|
+
# the chunk currently being encoded.
|
|
104
107
|
best_normal = matches.max_by { |m| [m.length, m.distance] }
|
|
105
108
|
|
|
106
109
|
if best_normal && best_normal.length >= nice_len &&
|
|
107
|
-
best_normal.distance <=
|
|
108
|
-
# CRITICAL: Only use normal match if distance is within actual dictionary buffer
|
|
109
|
-
# This prevents invalid matches that reference bytes not yet written to dictionary
|
|
110
|
+
best_normal.distance <= position
|
|
110
111
|
# Use normal match
|
|
111
112
|
return [best_normal.distance + REPS, best_normal.length]
|
|
112
113
|
end
|
|
@@ -116,10 +117,8 @@ module Omnizip
|
|
|
116
117
|
if rep_len >= 2 && rep_len >= (best_normal&.length || 0)
|
|
117
118
|
# Use repeated match
|
|
118
119
|
[rep_index, rep_len]
|
|
119
|
-
elsif best_normal && best_normal.distance <=
|
|
120
|
-
#
|
|
121
|
-
# Use dictionary.buffer.bytesize (actual data), NOT dictionary.size (max capacity)
|
|
122
|
-
# This prevents invalid matches that reference bytes not yet written to dictionary
|
|
120
|
+
elsif best_normal && best_normal.distance <= position
|
|
121
|
+
# Use normal match (any length beats encoding literals)
|
|
123
122
|
[best_normal.distance + REPS, best_normal.length]
|
|
124
123
|
else
|
|
125
124
|
# Use literal - return UINT32_MAX to indicate literal (not 0!)
|
|
@@ -136,7 +136,7 @@ module Omnizip
|
|
|
136
136
|
# @return [IO] The output_stream (for chaining)
|
|
137
137
|
def decompress(input_stream, output_stream, options = nil)
|
|
138
138
|
# Set binary encoding on output stream for proper byte handling
|
|
139
|
-
output_stream.set_encoding(Encoding::BINARY)
|
|
139
|
+
output_stream.set_encoding(Encoding::BINARY)
|
|
140
140
|
|
|
141
141
|
# Build decoder options, merging with instance variables as fallbacks
|
|
142
142
|
decoder_opts = build_decoder_options(options)
|
|
@@ -152,7 +152,11 @@ module Omnizip
|
|
|
152
152
|
comp_size & 0xFF, # Compressed size low byte (BIG-ENDIAN)
|
|
153
153
|
].pack("CCCCC")
|
|
154
154
|
|
|
155
|
-
|
|
155
|
+
# The properties byte is only present when the control byte
|
|
156
|
+
# announces it (reset level >= 2, i.e. 0xC0/0xE0). Writing it for
|
|
157
|
+
# plain continuation chunks (0x80) desynchronizes the decoder,
|
|
158
|
+
# which does not expect a byte there.
|
|
159
|
+
prop_bytes = @need_props && @properties ? [@properties].pack("C") : ""
|
|
156
160
|
header + prop_bytes + @compressed_data
|
|
157
161
|
end
|
|
158
162
|
end
|
|
@@ -83,23 +83,19 @@ module Omnizip
|
|
|
83
83
|
)
|
|
84
84
|
|
|
85
85
|
# Read input
|
|
86
|
-
input_data =
|
|
86
|
+
input_data = Omnizip::IO::Source.for(input).read
|
|
87
87
|
|
|
88
88
|
# Encode with LZMA2
|
|
89
89
|
compressed = encoder.encode(input_data)
|
|
90
90
|
|
|
91
91
|
# Write to output
|
|
92
|
-
|
|
93
|
-
output.write(compressed)
|
|
94
|
-
else
|
|
95
|
-
output.replace(compressed)
|
|
96
|
-
end
|
|
92
|
+
Omnizip::IO::Sink.for(output).write(compressed)
|
|
97
93
|
end
|
|
98
94
|
|
|
99
95
|
# Decompress LZMA2 data
|
|
100
96
|
def decompress(input, output, options = {})
|
|
101
97
|
# Read input data
|
|
102
|
-
input_data =
|
|
98
|
+
input_data = Omnizip::IO::Source.for(input).read
|
|
103
99
|
input_stream = StringIO.new(input_data)
|
|
104
100
|
input_stream.set_encoding(Encoding::BINARY)
|
|
105
101
|
|
|
@@ -120,11 +116,7 @@ module Omnizip
|
|
|
120
116
|
decompressed = decoder.decode_stream
|
|
121
117
|
|
|
122
118
|
# Write to output
|
|
123
|
-
|
|
124
|
-
output.write(decompressed)
|
|
125
|
-
else
|
|
126
|
-
output.replace(decompressed)
|
|
127
|
-
end
|
|
119
|
+
Omnizip::IO::Sink.for(output).write(decompressed)
|
|
128
120
|
end
|
|
129
121
|
|
|
130
122
|
# Encode dictionary size as single byte for LZMA2 properties
|
|
@@ -73,10 +73,12 @@ module Omnizip
|
|
|
73
73
|
HUFFMAN_MAX_LOG = 11
|
|
74
74
|
HUFFMAN_MAX_CODE_LENGTH = 11
|
|
75
75
|
HUFFMAN_STANDARD_TABLE_SIZE = 256
|
|
76
|
+
HUF_SYMBOLVALUE_MAX = 255
|
|
76
77
|
|
|
77
78
|
# FSE Table Limits (RFC 8878 Section 4.1)
|
|
78
79
|
FSE_MAX_ACCURACY_LOG = 9
|
|
79
80
|
FSE_MIN_ACCURACY_LOG = 5
|
|
81
|
+
FSE_DEFAULT_TABLELOG = 6
|
|
80
82
|
|
|
81
83
|
# Compression levels
|
|
82
84
|
MIN_LEVEL = 1
|
|
@@ -86,55 +88,66 @@ module Omnizip
|
|
|
86
88
|
# Buffer size for streaming operations
|
|
87
89
|
BUFFER_SIZE = 128 * 1024 # 128KB
|
|
88
90
|
|
|
89
|
-
# Literal length codes (RFC 8878 Table
|
|
91
|
+
# Literal length codes (RFC 8878 Table: literals length codes)
|
|
90
92
|
# Each entry: [baseline, extra_bits]
|
|
91
93
|
LITERAL_LENGTH_TABLE = [
|
|
92
94
|
[0, 0], [1, 0], [2, 0], [3, 0], [4, 0], [5, 0], [6, 0], [7, 0],
|
|
93
95
|
[8, 0], [9, 0], [10, 0], [11, 0], [12, 0], [13, 0], [14, 0], [15, 0],
|
|
94
|
-
[16, 1], [18, 1], [20, 1], [22, 1], [24,
|
|
95
|
-
[48,
|
|
96
|
-
[4096,
|
|
96
|
+
[16, 1], [18, 1], [20, 1], [22, 1], [24, 2], [28, 2], [32, 3], [40, 3],
|
|
97
|
+
[48, 4], [64, 6], [128, 7], [256, 8], [512, 9], [1024, 10], [2048, 11],
|
|
98
|
+
[4096, 12], [8192, 13], [16384, 14], [32768, 15], [65536, 16]
|
|
97
99
|
].freeze
|
|
98
100
|
|
|
99
|
-
# Match length codes (RFC 8878 Table
|
|
101
|
+
# Match length codes (RFC 8878 Table: match length codes, 53 entries)
|
|
100
102
|
# Each entry: [baseline, extra_bits]
|
|
101
103
|
MATCH_LENGTH_TABLE = [
|
|
102
104
|
[3, 0], [4, 0], [5, 0], [6, 0], [7, 0], [8, 0], [9, 0], [10, 0],
|
|
103
105
|
[11, 0], [12, 0], [13, 0], [14, 0], [15, 0], [16, 0], [17, 0], [18, 0],
|
|
104
106
|
[19, 0], [20, 0], [21, 0], [22, 0], [23, 0], [24, 0], [25, 0], [26, 0],
|
|
105
107
|
[27, 0], [28, 0], [29, 0], [30, 0], [31, 0], [32, 0], [33, 0], [34, 0],
|
|
106
|
-
[35, 1], [37, 1], [39, 1], [41, 1], [43,
|
|
107
|
-
[67,
|
|
108
|
-
[
|
|
109
|
-
[835, 2], [899, 2], [963, 2], [1027, 2], [1283, 2], [1539, 2],
|
|
110
|
-
[1795, 2], [2051, 2], [2307, 2], [2563, 2]
|
|
108
|
+
[35, 1], [37, 1], [39, 1], [41, 1], [43, 2], [47, 2], [51, 3], [59, 3],
|
|
109
|
+
[67, 4], [83, 4], [99, 5], [131, 7], [259, 8], [515, 9], [1027, 10],
|
|
110
|
+
[2051, 11], [4099, 12], [8195, 13], [16387, 14], [32771, 15], [65539, 16]
|
|
111
111
|
].freeze
|
|
112
112
|
|
|
113
|
-
#
|
|
113
|
+
# Offset codes (RFC 8878 Section 3.1.2.2.3.2)
|
|
114
|
+
# OF_BASE[c] + read(c bits) = match distance for c >= 2; codes 0 and
|
|
115
|
+
# 1 are the repeat-offset specials handled by the executor.
|
|
116
|
+
OF_BASE = [
|
|
117
|
+
0, 1, 1, 5, 13, 29, 61, 125, 253, 509, 1021, 2045, 4093, 8189,
|
|
118
|
+
16381, 32765, 65533, 131069, 262141, 524285, 1048573, 2097149,
|
|
119
|
+
4194301, 8388605, 16777213, 33554429, 67108861, 134217725,
|
|
120
|
+
268435453, 536870909, 1073741821, 2147483645
|
|
121
|
+
].freeze
|
|
122
|
+
OF_BITS = (0..31).to_a.freeze
|
|
123
|
+
|
|
124
|
+
# Predefined FSE distributions (RFC 8878 Section 4.1.3, matching
|
|
125
|
+
# the zstd C reference). -1 marks a "less than 1" probability that
|
|
126
|
+
# occupies a single cell at the top of the table.
|
|
114
127
|
PREDEFINED_LL_DISTRIBUTION = [
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
128
|
+
4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1,
|
|
129
|
+
2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1,
|
|
130
|
+
-1, -1, -1, -1
|
|
118
131
|
].freeze
|
|
119
132
|
|
|
120
|
-
#
|
|
121
|
-
# Sum = 64 (must equal 2^6 = 64)
|
|
133
|
+
# Sum: 4+3+2*11+1*3 + 2*8+3+2*2+1*5 = 32 + 32 = 64 = 2^6
|
|
122
134
|
PREDEFINED_ML_DISTRIBUTION = [
|
|
123
135
|
1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
|
|
124
136
|
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
|
125
|
-
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
|
126
|
-
1, 1, 1, 1,
|
|
127
|
-
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
128
|
-
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
129
|
-
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
130
|
-
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
|
|
137
|
+
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1,
|
|
138
|
+
-1, -1, -1, -1, -1
|
|
131
139
|
].freeze
|
|
132
140
|
|
|
133
|
-
#
|
|
141
|
+
# Sum: 27 positive + 5 low-probability = 32 = 2^5
|
|
134
142
|
PREDEFINED_OFFSET_DISTRIBUTION = [
|
|
135
|
-
1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1,
|
|
136
|
-
|
|
143
|
+
1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
|
|
144
|
+
1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1, 0, 0, 0
|
|
137
145
|
].freeze
|
|
146
|
+
|
|
147
|
+
# floor(log2(x)); 0 for x == 0 (C ZSTD_highbit32 defensive value)
|
|
148
|
+
def self.highbit32(value)
|
|
149
|
+
value.zero? ? 0 : value.bit_length - 1
|
|
150
|
+
end
|
|
138
151
|
end
|
|
139
152
|
end
|
|
140
153
|
end
|
|
@@ -23,61 +23,57 @@
|
|
|
23
23
|
module Omnizip
|
|
24
24
|
module Algorithms
|
|
25
25
|
class Zstandard
|
|
26
|
-
# Pure Ruby Zstandard decoder (RFC 8878)
|
|
26
|
+
# Pure Ruby Zstandard decoder (RFC 8878).
|
|
27
27
|
#
|
|
28
|
-
#
|
|
28
|
+
# Pipeline:
|
|
29
29
|
#
|
|
30
|
-
#
|
|
31
|
-
#
|
|
32
|
-
#
|
|
33
|
-
#
|
|
34
|
-
#
|
|
35
|
-
#
|
|
36
|
-
#
|
|
37
|
-
#
|
|
30
|
+
# 1. For each frame: parse the frame header and reset per-frame
|
|
31
|
+
# state (repeat offsets, previous Huffman table, previous FSE
|
|
32
|
+
# tables).
|
|
33
|
+
# 2. For each block: parse the 3-byte block header and dispatch on
|
|
34
|
+
# the block type (raw copy, RLE expansion, or compressed =
|
|
35
|
+
# literals + sequences).
|
|
36
|
+
# 3. Verify the optional content checksum: the low 32 bits of
|
|
37
|
+
# XXHash64 over the decoded frame content.
|
|
38
38
|
class Decoder
|
|
39
39
|
include Constants
|
|
40
40
|
|
|
41
|
-
# @return [IO]
|
|
41
|
+
# @return [IO] input stream
|
|
42
42
|
attr_reader :input_stream
|
|
43
43
|
|
|
44
|
-
# Initialize decoder
|
|
45
|
-
#
|
|
46
|
-
# @param input_stream [IO] Input stream of compressed data
|
|
47
44
|
def initialize(input_stream)
|
|
48
45
|
@input_stream = input_stream
|
|
49
|
-
@
|
|
46
|
+
@executor = SequenceExecutor.new
|
|
50
47
|
@previous_huffman_table = nil
|
|
51
48
|
@previous_fse_tables = {}
|
|
52
49
|
end
|
|
53
50
|
|
|
54
|
-
# Decode
|
|
51
|
+
# Decode a complete stream (one or more concatenated frames).
|
|
55
52
|
#
|
|
56
|
-
# @return [String]
|
|
53
|
+
# @return [String] decompressed data (binary)
|
|
57
54
|
def decode_stream
|
|
55
|
+
data = read_all
|
|
58
56
|
output = String.new(encoding: Encoding::BINARY)
|
|
57
|
+
pos = 0
|
|
59
58
|
|
|
60
59
|
loop do
|
|
61
|
-
|
|
62
|
-
|
|
60
|
+
remaining = data.bytesize - pos
|
|
61
|
+
break if remaining.zero?
|
|
62
|
+
raise Omnizip::DecompressionError, "trailing bytes are not a frame" if remaining < 4
|
|
63
|
+
|
|
64
|
+
magic = data.byteslice(pos, 4).unpack1("V")
|
|
63
65
|
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
skip_frame
|
|
66
|
+
if (magic & SKIPPABLE_MAGIC_MASK) == SKIPPABLE_MAGIC_BASE
|
|
67
|
+
pos = skip_skippable_frame(data, pos)
|
|
67
68
|
next
|
|
68
69
|
end
|
|
69
|
-
|
|
70
|
-
# Validate magic number
|
|
71
70
|
unless magic == MAGIC_NUMBER
|
|
72
|
-
raise
|
|
71
|
+
raise Omnizip::DecompressionError,
|
|
72
|
+
"invalid Zstandard magic: 0x#{magic.to_s(16)}"
|
|
73
73
|
end
|
|
74
74
|
|
|
75
|
-
|
|
76
|
-
frame_output = decode_frame
|
|
75
|
+
frame_output, pos = decode_frame(data, pos + 4)
|
|
77
76
|
output << frame_output
|
|
78
|
-
|
|
79
|
-
# Check for more frames
|
|
80
|
-
break if @input_stream.eof?
|
|
81
77
|
end
|
|
82
78
|
|
|
83
79
|
output
|
|
@@ -85,139 +81,108 @@ module Omnizip
|
|
|
85
81
|
|
|
86
82
|
private
|
|
87
83
|
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
84
|
+
def read_all
|
|
85
|
+
data = @input_stream.read
|
|
86
|
+
data ||= ""
|
|
87
|
+
data.dup.force_encoding(Encoding::BINARY)
|
|
91
88
|
end
|
|
92
89
|
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
# Read frame size (4 bytes)
|
|
96
|
-
size = read_u32le
|
|
97
|
-
@input_stream.seek(size, IO::SEEK_CUR)
|
|
98
|
-
end
|
|
90
|
+
def skip_skippable_frame(data, pos)
|
|
91
|
+
raise Omnizip::DecompressionError, "truncated skippable frame" if data.bytesize < pos + 8
|
|
99
92
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
93
|
+
size = data.byteslice(pos + 4, 4).unpack1("V")
|
|
94
|
+
end_pos = pos + 8 + size
|
|
95
|
+
if data.bytesize < end_pos
|
|
96
|
+
raise Omnizip::DecompressionError, "truncated skippable frame body"
|
|
97
|
+
end
|
|
104
98
|
|
|
105
|
-
|
|
99
|
+
end_pos
|
|
106
100
|
end
|
|
107
101
|
|
|
108
|
-
# Decode
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
header = Frame::Header.
|
|
102
|
+
# Decode one frame (input positioned after the magic).
|
|
103
|
+
# Returns [frame_output, pos_after_frame].
|
|
104
|
+
def decode_frame(data, pos)
|
|
105
|
+
header, pos = Frame::Header.parse_from(data, pos)
|
|
112
106
|
|
|
113
|
-
#
|
|
114
|
-
|
|
107
|
+
# Reset per-frame state: repeat offsets, previous Huffman
|
|
108
|
+
# table, previous FSE tables.
|
|
109
|
+
@executor = SequenceExecutor.new
|
|
110
|
+
@previous_huffman_table = nil
|
|
111
|
+
@previous_fse_tables = {}
|
|
115
112
|
|
|
116
|
-
# Decode blocks
|
|
117
113
|
output = String.new(encoding: Encoding::BINARY)
|
|
118
|
-
|
|
119
114
|
loop do
|
|
120
|
-
block = Frame::Block.
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
115
|
+
block, pos = Frame::Block.parse_from(data, pos)
|
|
116
|
+
if block.reserved?
|
|
117
|
+
raise Omnizip::DecompressionError,
|
|
118
|
+
"reserved block type in frame"
|
|
119
|
+
end
|
|
125
120
|
|
|
121
|
+
pos = decode_block(block, data, pos, output)
|
|
126
122
|
break if block.last_block
|
|
127
123
|
end
|
|
128
124
|
|
|
129
|
-
# Verify checksum if present
|
|
130
125
|
if header.content_checksum?
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
output
|
|
135
|
-
end
|
|
126
|
+
if data.bytesize < pos + 4
|
|
127
|
+
raise Omnizip::DecompressionError, "truncated frame checksum"
|
|
128
|
+
end
|
|
136
129
|
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
130
|
+
expected = data.byteslice(pos, 4).unpack1("V")
|
|
131
|
+
actual = XXHash64.frame_checksum(output)
|
|
132
|
+
if expected != actual
|
|
133
|
+
raise Omnizip::DecompressionError,
|
|
134
|
+
"frame checksum mismatch: stored 0x#{expected.to_s(16)}, " \
|
|
135
|
+
"computed 0x#{actual.to_s(16)}"
|
|
136
|
+
end
|
|
137
|
+
pos += 4
|
|
138
|
+
end
|
|
141
139
|
|
|
142
|
-
|
|
140
|
+
[output, pos]
|
|
143
141
|
end
|
|
144
142
|
|
|
145
|
-
|
|
146
|
-
def decode_block(block, _header)
|
|
143
|
+
def decode_block(block, data, pos, output)
|
|
147
144
|
case block.block_type
|
|
148
145
|
when BLOCK_TYPE_RAW
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
when BLOCK_TYPE_COMPRESSED
|
|
153
|
-
decode_compressed_block(block)
|
|
154
|
-
else
|
|
155
|
-
raise "Reserved block type: #{block.block_type}"
|
|
156
|
-
end
|
|
157
|
-
end
|
|
158
|
-
|
|
159
|
-
# Decode raw (uncompressed) block
|
|
160
|
-
def decode_raw_block(block)
|
|
161
|
-
@input_stream.read(block.block_size)
|
|
162
|
-
end
|
|
163
|
-
|
|
164
|
-
# Decode RLE block
|
|
165
|
-
def decode_rle_block(block)
|
|
166
|
-
byte = @input_stream.read(1)
|
|
167
|
-
byte * block.block_size
|
|
168
|
-
end
|
|
169
|
-
|
|
170
|
-
# Decode compressed block
|
|
171
|
-
def decode_compressed_block(_block)
|
|
172
|
-
# Record start position for calculating remaining bytes
|
|
173
|
-
@input_stream.pos
|
|
174
|
-
|
|
175
|
-
# Decode literals section
|
|
176
|
-
literals_decoder = LiteralsDecoder.decode(@input_stream,
|
|
177
|
-
@previous_huffman_table)
|
|
178
|
-
literals = literals_decoder.literals
|
|
179
|
-
@previous_huffman_table = literals_decoder.huffman_table
|
|
180
|
-
|
|
181
|
-
# Decode sequences section
|
|
182
|
-
sequences_decoder = SequencesDecoder.decode(@input_stream,
|
|
183
|
-
literals.bytesize,
|
|
184
|
-
@previous_fse_tables)
|
|
185
|
-
sequences = sequences_decoder.sequences
|
|
186
|
-
|
|
187
|
-
# Execute sequences to produce output
|
|
188
|
-
if sequences.empty?
|
|
189
|
-
# No sequences - literals are the output
|
|
190
|
-
literals
|
|
191
|
-
else
|
|
192
|
-
SequenceExecutor.execute(literals, sequences)
|
|
193
|
-
end
|
|
194
|
-
end
|
|
146
|
+
if data.bytesize < pos + block.block_size
|
|
147
|
+
raise Omnizip::DecompressionError, "truncated raw block"
|
|
148
|
+
end
|
|
195
149
|
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
return unless checksum_bytes && checksum_bytes.length == 4
|
|
150
|
+
output << data.byteslice(pos, block.block_size)
|
|
151
|
+
pos + block.block_size
|
|
152
|
+
when BLOCK_TYPE_RLE
|
|
153
|
+
raise Omnizip::DecompressionError, "truncated RLE block" if data.bytesize < pos + 1
|
|
201
154
|
|
|
202
|
-
|
|
203
|
-
|
|
155
|
+
output << (data.getbyte(pos).chr * block.block_size)
|
|
156
|
+
pos + 1
|
|
157
|
+
when BLOCK_TYPE_COMPRESSED
|
|
158
|
+
block_end = pos + block.block_size
|
|
159
|
+
if data.bytesize < block_end
|
|
160
|
+
raise Omnizip::DecompressionError, "truncated compressed block"
|
|
161
|
+
end
|
|
204
162
|
|
|
205
|
-
|
|
206
|
-
|
|
163
|
+
decode_compressed_block(data.byteslice(pos...block_end), output)
|
|
164
|
+
block_end
|
|
207
165
|
end
|
|
208
166
|
end
|
|
209
167
|
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
168
|
+
def decode_compressed_block(block_input, output)
|
|
169
|
+
literals_decoder = LiteralsDecoder.decode(block_input,
|
|
170
|
+
@previous_huffman_table)
|
|
171
|
+
@previous_huffman_table = literals_decoder.huffman_table ||
|
|
172
|
+
@previous_huffman_table
|
|
173
|
+
|
|
174
|
+
sequences_decoder = SequencesDecoder.decode(
|
|
175
|
+
block_input.byteslice(literals_decoder.consumed..),
|
|
176
|
+
@previous_fse_tables,
|
|
177
|
+
@executor,
|
|
178
|
+
)
|
|
179
|
+
# Merge: Repeat-mode tables are not re-emitted, so the entries
|
|
180
|
+
# carried from earlier blocks must survive.
|
|
181
|
+
@previous_fse_tables =
|
|
182
|
+
@previous_fse_tables.merge(sequences_decoder.fse_tables)
|
|
183
|
+
|
|
184
|
+
@executor.execute(literals_decoder.literals,
|
|
185
|
+
sequences_decoder.sequences, output)
|
|
221
186
|
end
|
|
222
187
|
end
|
|
223
188
|
end
|