omnizip 0.3.13 → 0.3.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. checksums.yaml +4 -4
  2. data/lib/omnizip/algorithm.rb +36 -1
  3. data/lib/omnizip/algorithms/bzip2.rb +1 -3
  4. data/lib/omnizip/algorithms/deflate.rb +1 -3
  5. data/lib/omnizip/algorithms/deflate64.rb +2 -2
  6. data/lib/omnizip/algorithms/lzma/optimal_encoder.rb +7 -8
  7. data/lib/omnizip/algorithms/lzma.rb +1 -1
  8. data/lib/omnizip/algorithms/lzma2/lzma2_chunk.rb +5 -1
  9. data/lib/omnizip/algorithms/lzma2.rb +4 -12
  10. data/lib/omnizip/algorithms/zstandard/constants.rb +38 -25
  11. data/lib/omnizip/algorithms/zstandard/decoder.rb +101 -136
  12. data/lib/omnizip/algorithms/zstandard/encoder.rb +81 -138
  13. data/lib/omnizip/algorithms/zstandard/frame/block.rb +17 -0
  14. data/lib/omnizip/algorithms/zstandard/frame/header.rb +42 -29
  15. data/lib/omnizip/algorithms/zstandard/fse/bitstream.rb +114 -117
  16. data/lib/omnizip/algorithms/zstandard/fse/encoder.rb +434 -222
  17. data/lib/omnizip/algorithms/zstandard/fse/interleaved.rb +92 -0
  18. data/lib/omnizip/algorithms/zstandard/fse/table.rb +98 -176
  19. data/lib/omnizip/algorithms/zstandard/fse/table_description.rb +203 -0
  20. data/lib/omnizip/algorithms/zstandard/fse.rb +21 -2
  21. data/lib/omnizip/algorithms/zstandard/huffman.rb +195 -188
  22. data/lib/omnizip/algorithms/zstandard/huffman_encoder.rb +232 -255
  23. data/lib/omnizip/algorithms/zstandard/literals.rb +170 -103
  24. data/lib/omnizip/algorithms/zstandard/literals_encoder.rb +47 -199
  25. data/lib/omnizip/algorithms/zstandard/sequences.rb +260 -252
  26. data/lib/omnizip/algorithms/zstandard/xxhash.rb +159 -0
  27. data/lib/omnizip/algorithms/zstandard.rb +20 -24
  28. data/lib/omnizip/algorithms.rb +14 -1
  29. data/lib/omnizip/buffer.rb +1 -2
  30. data/lib/omnizip/commands/archive_list_command.rb +9 -7
  31. data/lib/omnizip/convenience.rb +1 -2
  32. data/lib/omnizip/entry.rb +44 -0
  33. data/lib/omnizip/extraction/selective_extractor.rb +2 -10
  34. data/lib/omnizip/filter_pipeline.rb +1 -1
  35. data/lib/omnizip/filter_registry.rb +18 -16
  36. data/lib/omnizip/filters/bcj2.rb +1 -1
  37. data/lib/omnizip/filters/bcj_arm.rb +1 -1
  38. data/lib/omnizip/filters/bcj_arm64.rb +1 -1
  39. data/lib/omnizip/filters/bcj_ia64.rb +1 -1
  40. data/lib/omnizip/filters/bcj_ppc.rb +1 -1
  41. data/lib/omnizip/filters/bcj_sparc.rb +1 -1
  42. data/lib/omnizip/filters/bcj_x86.rb +1 -1
  43. data/lib/omnizip/filters.rb +0 -2
  44. data/lib/omnizip/formats/bzip2_file.rb +0 -7
  45. data/lib/omnizip/formats/cpio/entry.rb +6 -0
  46. data/lib/omnizip/formats/cpio.rb +0 -6
  47. data/lib/omnizip/formats/gzip.rb +0 -7
  48. data/lib/omnizip/formats/iso/directory_record.rb +7 -0
  49. data/lib/omnizip/formats/iso.rb +0 -5
  50. data/lib/omnizip/formats/lzip.rb +0 -7
  51. data/lib/omnizip/formats/lzma_alone.rb +0 -6
  52. data/lib/omnizip/formats/msi/entry.rb +6 -0
  53. data/lib/omnizip/formats/msi.rb +0 -9
  54. data/lib/omnizip/formats/ole/dirent.rb +6 -0
  55. data/lib/omnizip/formats/ole.rb +0 -9
  56. data/lib/omnizip/formats/rar.rb +0 -6
  57. data/lib/omnizip/formats/rar3/reader.rb +7 -0
  58. data/lib/omnizip/formats/rar5/reader.rb +7 -0
  59. data/lib/omnizip/formats/rpm/entry.rb +6 -0
  60. data/lib/omnizip/formats/seven_zip/models/file_entry.rb +7 -0
  61. data/lib/omnizip/formats/seven_zip.rb +0 -6
  62. data/lib/omnizip/formats/tar/entry.rb +6 -0
  63. data/lib/omnizip/formats/tar.rb +2 -10
  64. data/lib/omnizip/formats/xar/entry.rb +6 -0
  65. data/lib/omnizip/formats/xar.rb +0 -6
  66. data/lib/omnizip/formats/zip.rb +0 -6
  67. data/lib/omnizip/implementations/seven_zip/lzma2/encoder.rb +33 -31
  68. data/lib/omnizip/implementations/xz_utils/lzma2/encoder.rb +177 -163
  69. data/lib/omnizip/implementations.rb +18 -4
  70. data/lib/omnizip/io/source.rb +3 -0
  71. data/lib/omnizip/metadata/entry_metadata.rb +7 -0
  72. data/lib/omnizip/parallel/engine.rb +48 -0
  73. data/lib/omnizip/parallel/parallel_compressor.rb +5 -16
  74. data/lib/omnizip/parallel/parallel_extractor.rb +5 -16
  75. data/lib/omnizip/parallel.rb +1 -0
  76. data/lib/omnizip/profile/profile_registry.rb +1 -2
  77. data/lib/omnizip/registry.rb +0 -1
  78. data/lib/omnizip/rubyzip_compat.rb +0 -1
  79. data/lib/omnizip/temp/temp_file.rb +1 -2
  80. data/lib/omnizip/version.rb +1 -1
  81. data/lib/omnizip/zip/entry.rb +7 -0
  82. data/lib/omnizip/zip/file.rb +4 -8
  83. data/lib/omnizip.rb +1 -1
  84. metadata +7 -6
  85. data/lib/omnizip/filters/filter_base.rb +0 -6
  86. data/lib/omnizip/filters/registration.rb +0 -22
  87. data/lib/omnizip/filters/registry.rb +0 -111
  88. data/lib/omnizip/format_registry.rb +0 -100
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 3e86ca6410983afcc2259af21021ae0049d2b5f4def32975d9fa1568297417e3
4
- data.tar.gz: b52224b6ef3a5b2dab3f56c9117daf36b679ecda95983cafd62ddd7d7f05b35a
3
+ metadata.gz: 978a7f51735e4fd90dbb0432b933ccdd17009d74c682db94f6ee6730fe1f893a
4
+ data.tar.gz: c02f967a72d720edbbc204bd6b36dc60a4c4f773aed3e308800880351a32c218
5
5
  SHA512:
6
- metadata.gz: 3b74c21559d4c813b83c82d80957c97d3f57f0d512cdb1895a92f869bd27594f5c0b60ae108179431e50f52bb44bfbb795343facb2de8f853258e483f4b63e75
7
- data.tar.gz: ff4f463c54686edc825f514bb74c527d33a0a9319e2952ac628984392eb5acfb26ce77077a12ae711ddb44a419361a93e3336ae43d8c85a23d01e3efd445a029
6
+ metadata.gz: 6664a917d2c82bca2779e62a082e1462c1dd8c36f3f4777a47a871feafd4d3bb25e7ae4eb492623a823aed8755777fce93374075a502b0a8beabe2becf9aed05
7
+ data.tar.gz: ccf0ff7279dc03c4c5aec2fd6a679419c3fa275966e3b7f076e3bdd4e5890e2141664f6137df46681d29738dbf7fb25ef95e1ec3e450c0e3cfc7509dd9f759d7
@@ -23,6 +23,8 @@ module Omnizip
23
23
  # the required methods. Algorithms are automatically registered with the
24
24
  # AlgorithmRegistry when defined.
25
25
  class Algorithm
26
+ require "stringio"
27
+
26
28
  attr_reader :options, :filter
27
29
 
28
30
  # Initialize algorithm with options.
@@ -38,7 +40,7 @@ module Omnizip
38
40
  # The filter will be applied before compression and reversed after
39
41
  # decompression. Returns self for method chaining.
40
42
  #
41
- # @param filter [Filters::FilterBase, FilterPipeline] Filter or
43
+ # @param filter [Omnizip::Filter, FilterPipeline] Filter or
42
44
  # pipeline to use
43
45
  # @return [self] For method chaining
44
46
  def with_filter(filter)
@@ -82,6 +84,39 @@ module Omnizip
82
84
  "#{self} must implement .metadata"
83
85
  end
84
86
 
87
+ # Class-level convenience: compress `data` and return the
88
+ # compressed bytes. Carries the algorithm's streaming contract
89
+ # over a single StringIO so callers with the whole payload in
90
+ # memory (parallel compressors, small files) don't need to
91
+ # instantiate and wire streams themselves.
92
+ #
93
+ # @param data [String, IO] Input to compress
94
+ # @param options [Hash] Forwarded to the algorithm
95
+ # @return [String] Compressed bytes
96
+ def compress(data, **options)
97
+ instance = new(options)
98
+ input = data.is_a?(::IO) ? data : ::StringIO.new(data.to_s.b)
99
+ output = ::StringIO.new
100
+ output.set_encoding(Encoding::BINARY)
101
+ instance.compress(input, output, options)
102
+ output.string
103
+ end
104
+
105
+ # Class-level convenience: decompress `data` and return the
106
+ # uncompressed bytes. Mirror of +.compress+.
107
+ #
108
+ # @param data [String, IO] Input to decompress
109
+ # @param options [Hash] Forwarded to the algorithm
110
+ # @return [String] Decompressed bytes
111
+ def decompress(data, **options)
112
+ instance = new(options)
113
+ input = data.is_a?(::IO) ? data : ::StringIO.new(data.to_s.b)
114
+ output = ::StringIO.new
115
+ output.set_encoding(Encoding::BINARY)
116
+ instance.decompress(input, output, options)
117
+ output.string
118
+ end
119
+
85
120
  # Automatically register algorithm when inherited.
86
121
  #
87
122
  # This hook is called whenever a class inherits from Algorithm,
@@ -82,9 +82,7 @@ module Omnizip
82
82
  # @param options [Models::CompressionOptions] Decompression options
83
83
  # @return [void]
84
84
  def decompress(input_stream, output_stream, _options = nil)
85
- if output_stream.respond_to?(:set_encoding)
86
- output_stream.set_encoding(Encoding::BINARY)
87
- end
85
+ output_stream.set_encoding(Encoding::BINARY)
88
86
  decoder = Decoder.new(input_stream)
89
87
  decompressed = decoder.decode_stream
90
88
  output_stream.write(decompressed)
@@ -77,9 +77,7 @@ module Omnizip
77
77
  # @param options [Models::CompressionOptions] Decompression options
78
78
  # @return [void]
79
79
  def decompress(input_stream, output_stream, _options = nil)
80
- if output_stream.respond_to?(:set_encoding)
81
- output_stream.set_encoding(Encoding::BINARY)
82
- end
80
+ output_stream.set_encoding(Encoding::BINARY)
83
81
  decoder = Decoder.new(input_stream)
84
82
  decompressed = decoder.decode_stream
85
83
  output_stream.write(decompressed)
@@ -73,8 +73,8 @@ module Omnizip
73
73
  return if compressed.nil? || compressed.empty?
74
74
 
75
75
  # Set output to binary mode if it's a StringIO
76
- output.set_encoding(Encoding::BINARY) if output.respond_to?(:set_encoding)
77
- output.binmode if output.respond_to?(:binmode)
76
+ output.set_encoding(Encoding::BINARY)
77
+ output.binmode
78
78
 
79
79
  # Use Zlib::Inflate with maximum window size
80
80
  inflater = Zlib::Inflate.new(Zlib::MAX_WBITS)
@@ -100,13 +100,14 @@ module Omnizip
100
100
 
101
101
  # We didn't find a long enough repeated match.
102
102
  # Encode it as a normal match if the match length is at least nice_len
103
- # AND the match distance is within the dictionary buffer.
103
+ # AND the match distance stays within the data decoded so far.
104
+ # A match may reference anything before `position`: bytes already
105
+ # committed to the dictionary *and* bytes decoded earlier within
106
+ # the chunk currently being encoded.
104
107
  best_normal = matches.max_by { |m| [m.length, m.distance] }
105
108
 
106
109
  if best_normal && best_normal.length >= nice_len &&
107
- best_normal.distance <= match_finder.dictionary.buffer.bytesize
108
- # CRITICAL: Only use normal match if distance is within actual dictionary buffer
109
- # This prevents invalid matches that reference bytes not yet written to dictionary
110
+ best_normal.distance <= position
110
111
  # Use normal match
111
112
  return [best_normal.distance + REPS, best_normal.length]
112
113
  end
@@ -116,10 +117,8 @@ module Omnizip
116
117
  if rep_len >= 2 && rep_len >= (best_normal&.length || 0)
117
118
  # Use repeated match
118
119
  [rep_index, rep_len]
119
- elsif best_normal && best_normal.distance <= match_finder.dictionary.buffer.bytesize
120
- # CRITICAL: Only use normal match if distance is within actual dictionary buffer
121
- # Use dictionary.buffer.bytesize (actual data), NOT dictionary.size (max capacity)
122
- # This prevents invalid matches that reference bytes not yet written to dictionary
120
+ elsif best_normal && best_normal.distance <= position
121
+ # Use normal match (any length beats encoding literals)
123
122
  [best_normal.distance + REPS, best_normal.length]
124
123
  else
125
124
  # Use literal - return UINT32_MAX to indicate literal (not 0!)
@@ -136,7 +136,7 @@ module Omnizip
136
136
  # @return [IO] The output_stream (for chaining)
137
137
  def decompress(input_stream, output_stream, options = nil)
138
138
  # Set binary encoding on output stream for proper byte handling
139
- output_stream.set_encoding(Encoding::BINARY) if output_stream.respond_to?(:set_encoding)
139
+ output_stream.set_encoding(Encoding::BINARY)
140
140
 
141
141
  # Build decoder options, merging with instance variables as fallbacks
142
142
  decoder_opts = build_decoder_options(options)
@@ -152,7 +152,11 @@ module Omnizip
152
152
  comp_size & 0xFF, # Compressed size low byte (BIG-ENDIAN)
153
153
  ].pack("CCCCC")
154
154
 
155
- prop_bytes = @properties ? [@properties].pack("C") : ""
155
+ # The properties byte is only present when the control byte
156
+ # announces it (reset level >= 2, i.e. 0xC0/0xE0). Writing it for
157
+ # plain continuation chunks (0x80) desynchronizes the decoder,
158
+ # which does not expect a byte there.
159
+ prop_bytes = @need_props && @properties ? [@properties].pack("C") : ""
156
160
  header + prop_bytes + @compressed_data
157
161
  end
158
162
  end
@@ -83,23 +83,19 @@ module Omnizip
83
83
  )
84
84
 
85
85
  # Read input
86
- input_data = input.respond_to?(:read) ? input.read : input
86
+ input_data = Omnizip::IO::Source.for(input).read
87
87
 
88
88
  # Encode with LZMA2
89
89
  compressed = encoder.encode(input_data)
90
90
 
91
91
  # Write to output
92
- if output.respond_to?(:write)
93
- output.write(compressed)
94
- else
95
- output.replace(compressed)
96
- end
92
+ Omnizip::IO::Sink.for(output).write(compressed)
97
93
  end
98
94
 
99
95
  # Decompress LZMA2 data
100
96
  def decompress(input, output, options = {})
101
97
  # Read input data
102
- input_data = input.respond_to?(:read) ? input.read : input
98
+ input_data = Omnizip::IO::Source.for(input).read
103
99
  input_stream = StringIO.new(input_data)
104
100
  input_stream.set_encoding(Encoding::BINARY)
105
101
 
@@ -120,11 +116,7 @@ module Omnizip
120
116
  decompressed = decoder.decode_stream
121
117
 
122
118
  # Write to output
123
- if output.respond_to?(:write)
124
- output.write(decompressed)
125
- else
126
- output.replace(decompressed)
127
- end
119
+ Omnizip::IO::Sink.for(output).write(decompressed)
128
120
  end
129
121
 
130
122
  # Encode dictionary size as single byte for LZMA2 properties
@@ -73,10 +73,12 @@ module Omnizip
73
73
  HUFFMAN_MAX_LOG = 11
74
74
  HUFFMAN_MAX_CODE_LENGTH = 11
75
75
  HUFFMAN_STANDARD_TABLE_SIZE = 256
76
+ HUF_SYMBOLVALUE_MAX = 255
76
77
 
77
78
  # FSE Table Limits (RFC 8878 Section 4.1)
78
79
  FSE_MAX_ACCURACY_LOG = 9
79
80
  FSE_MIN_ACCURACY_LOG = 5
81
+ FSE_DEFAULT_TABLELOG = 6
80
82
 
81
83
  # Compression levels
82
84
  MIN_LEVEL = 1
@@ -86,55 +88,66 @@ module Omnizip
86
88
  # Buffer size for streaming operations
87
89
  BUFFER_SIZE = 128 * 1024 # 128KB
88
90
 
89
- # Literal length codes (RFC 8878 Table 9)
91
+ # Literal length codes (RFC 8878 Table: literals length codes)
90
92
  # Each entry: [baseline, extra_bits]
91
93
  LITERAL_LENGTH_TABLE = [
92
94
  [0, 0], [1, 0], [2, 0], [3, 0], [4, 0], [5, 0], [6, 0], [7, 0],
93
95
  [8, 0], [9, 0], [10, 0], [11, 0], [12, 0], [13, 0], [14, 0], [15, 0],
94
- [16, 1], [18, 1], [20, 1], [22, 1], [24, 1], [28, 1], [32, 1], [40, 1],
95
- [48, 1], [64, 1], [128, 2], [256, 2], [512, 2], [1024, 2], [2048, 2],
96
- [4096, 2], [8192, 2], [16384, 3], [32768, 3], [65536, 3]
96
+ [16, 1], [18, 1], [20, 1], [22, 1], [24, 2], [28, 2], [32, 3], [40, 3],
97
+ [48, 4], [64, 6], [128, 7], [256, 8], [512, 9], [1024, 10], [2048, 11],
98
+ [4096, 12], [8192, 13], [16384, 14], [32768, 15], [65536, 16]
97
99
  ].freeze
98
100
 
99
- # Match length codes (RFC 8878 Table 10)
101
+ # Match length codes (RFC 8878 Table: match length codes, 53 entries)
100
102
  # Each entry: [baseline, extra_bits]
101
103
  MATCH_LENGTH_TABLE = [
102
104
  [3, 0], [4, 0], [5, 0], [6, 0], [7, 0], [8, 0], [9, 0], [10, 0],
103
105
  [11, 0], [12, 0], [13, 0], [14, 0], [15, 0], [16, 0], [17, 0], [18, 0],
104
106
  [19, 0], [20, 0], [21, 0], [22, 0], [23, 0], [24, 0], [25, 0], [26, 0],
105
107
  [27, 0], [28, 0], [29, 0], [30, 0], [31, 0], [32, 0], [33, 0], [34, 0],
106
- [35, 1], [37, 1], [39, 1], [41, 1], [43, 1], [47, 1], [51, 1], [59, 1],
107
- [67, 1], [83, 1], [99, 1], [131, 2], [195, 2], [259, 2], [323, 2],
108
- [387, 2], [451, 2], [515, 2], [579, 2], [643, 2], [707, 2], [771, 2],
109
- [835, 2], [899, 2], [963, 2], [1027, 2], [1283, 2], [1539, 2],
110
- [1795, 2], [2051, 2], [2307, 2], [2563, 2]
108
+ [35, 1], [37, 1], [39, 1], [41, 1], [43, 2], [47, 2], [51, 3], [59, 3],
109
+ [67, 4], [83, 4], [99, 5], [131, 7], [259, 8], [515, 9], [1027, 10],
110
+ [2051, 11], [4099, 12], [8195, 13], [16387, 14], [32771, 15], [65539, 16]
111
111
  ].freeze
112
112
 
113
- # Predefined FSE distribution for literals length (RFC 8878 Section 4.1.3)
113
+ # Offset codes (RFC 8878 Section 3.1.2.2.3.2)
114
+ # OF_BASE[c] + read(c bits) = match distance for c >= 2; codes 0 and
115
+ # 1 are the repeat-offset specials handled by the executor.
116
+ OF_BASE = [
117
+ 0, 1, 1, 5, 13, 29, 61, 125, 253, 509, 1021, 2045, 4093, 8189,
118
+ 16381, 32765, 65533, 131069, 262141, 524285, 1048573, 2097149,
119
+ 4194301, 8388605, 16777213, 33554429, 67108861, 134217725,
120
+ 268435453, 536870909, 1073741821, 2147483645
121
+ ].freeze
122
+ OF_BITS = (0..31).to_a.freeze
123
+
124
+ # Predefined FSE distributions (RFC 8878 Section 4.1.3, matching
125
+ # the zstd C reference). -1 marks a "less than 1" probability that
126
+ # occupies a single cell at the top of the table.
114
127
  PREDEFINED_LL_DISTRIBUTION = [
115
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
116
- 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
117
- 0, 0, 0, 0
128
+ 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1,
129
+ 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1,
130
+ -1, -1, -1, -1
118
131
  ].freeze
119
132
 
120
- # Predefined FSE distribution for match length (RFC 8878 Section 4.1.3)
121
- # Sum = 64 (must equal 2^6 = 64)
133
+ # Sum: 4+3+2*11+1*3 + 2*8+3+2*2+1*5 = 32 + 32 = 64 = 2^6
122
134
  PREDEFINED_ML_DISTRIBUTION = [
123
135
  1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
124
136
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
125
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
126
- 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
127
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
128
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
129
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
130
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
137
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1,
138
+ -1, -1, -1, -1, -1
131
139
  ].freeze
132
140
 
133
- # Predefined FSE distribution for offset (RFC 8878 Section 4.1.3)
141
+ # Sum: 27 positive + 5 low-probability = 32 = 2^5
134
142
  PREDEFINED_OFFSET_DISTRIBUTION = [
135
- 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 0,
136
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
143
+ 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
144
+ 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1, 0, 0, 0
137
145
  ].freeze
146
+
147
+ # floor(log2(x)); 0 for x == 0 (C ZSTD_highbit32 defensive value)
148
+ def self.highbit32(value)
149
+ value.zero? ? 0 : value.bit_length - 1
150
+ end
138
151
  end
139
152
  end
140
153
  end
@@ -23,61 +23,57 @@
23
23
  module Omnizip
24
24
  module Algorithms
25
25
  class Zstandard
26
- # Pure Ruby Zstandard decoder (RFC 8878)
26
+ # Pure Ruby Zstandard decoder (RFC 8878).
27
27
  #
28
- # Decodes Zstandard-compressed data according to RFC 8878.
28
+ # Pipeline:
29
29
  #
30
- # Decoder pipeline:
31
- # 1. Parse frame header
32
- # 2. For each block:
33
- # a. Parse block header
34
- # b. Decode literals section
35
- # c. Decode sequences section
36
- # d. Execute sequences (LZ77 copy operations)
37
- # 3. Verify content checksum if present
30
+ # 1. For each frame: parse the frame header and reset per-frame
31
+ # state (repeat offsets, previous Huffman table, previous FSE
32
+ # tables).
33
+ # 2. For each block: parse the 3-byte block header and dispatch on
34
+ # the block type (raw copy, RLE expansion, or compressed =
35
+ # literals + sequences).
36
+ # 3. Verify the optional content checksum: the low 32 bits of
37
+ # XXHash64 over the decoded frame content.
38
38
  class Decoder
39
39
  include Constants
40
40
 
41
- # @return [IO] Input stream
41
+ # @return [IO] input stream
42
42
  attr_reader :input_stream
43
43
 
44
- # Initialize decoder
45
- #
46
- # @param input_stream [IO] Input stream of compressed data
47
44
  def initialize(input_stream)
48
45
  @input_stream = input_stream
49
- @repeat_offsets = DEFAULT_REPEAT_OFFSETS.dup
46
+ @executor = SequenceExecutor.new
50
47
  @previous_huffman_table = nil
51
48
  @previous_fse_tables = {}
52
49
  end
53
50
 
54
- # Decode compressed data stream
51
+ # Decode a complete stream (one or more concatenated frames).
55
52
  #
56
- # @return [String] Decompressed data
53
+ # @return [String] decompressed data (binary)
57
54
  def decode_stream
55
+ data = read_all
58
56
  output = String.new(encoding: Encoding::BINARY)
57
+ pos = 0
59
58
 
60
59
  loop do
61
- # Read magic number
62
- magic = read_u32le
60
+ remaining = data.bytesize - pos
61
+ break if remaining.zero?
62
+ raise Omnizip::DecompressionError, "trailing bytes are not a frame" if remaining < 4
63
+
64
+ magic = data.byteslice(pos, 4).unpack1("V")
63
65
 
64
- # Check for skippable frame
65
- if skippable_frame?(magic)
66
- skip_frame
66
+ if (magic & SKIPPABLE_MAGIC_MASK) == SKIPPABLE_MAGIC_BASE
67
+ pos = skip_skippable_frame(data, pos)
67
68
  next
68
69
  end
69
-
70
- # Validate magic number
71
70
  unless magic == MAGIC_NUMBER
72
- raise "Invalid Zstandard magic: 0x#{magic.to_s(16)}"
71
+ raise Omnizip::DecompressionError,
72
+ "invalid Zstandard magic: 0x#{magic.to_s(16)}"
73
73
  end
74
74
 
75
- # Parse frame
76
- frame_output = decode_frame
75
+ frame_output, pos = decode_frame(data, pos + 4)
77
76
  output << frame_output
78
-
79
- # Check for more frames
80
- break if @input_stream.eof?
81
77
  end
82
78
 
83
79
  output
@@ -85,139 +81,108 @@ module Omnizip
85
81
 
86
82
  private
87
83
 
88
- # Check if frame is skippable
89
- def skippable_frame?(magic)
90
- (magic & SKIPPABLE_MAGIC_MASK) == SKIPPABLE_MAGIC_BASE
84
+ def read_all
85
+ data = @input_stream.read
86
+ data ||= ""
87
+ data.dup.force_encoding(Encoding::BINARY)
91
88
  end
92
89
 
93
- # Skip skippable frame
94
- def skip_frame
95
- # Read frame size (4 bytes)
96
- size = read_u32le
97
- @input_stream.seek(size, IO::SEEK_CUR)
98
- end
90
+ def skip_skippable_frame(data, pos)
91
+ raise Omnizip::DecompressionError, "truncated skippable frame" if data.bytesize < pos + 8
99
92
 
100
- # Read unsigned 32-bit little-endian
101
- def read_u32le
102
- bytes = @input_stream.read(4)
103
- return 0 if bytes.nil? || bytes.length < 4
93
+ size = data.byteslice(pos + 4, 4).unpack1("V")
94
+ end_pos = pos + 8 + size
95
+ if data.bytesize < end_pos
96
+ raise Omnizip::DecompressionError, "truncated skippable frame body"
97
+ end
104
98
 
105
- bytes.unpack1("V")
99
+ end_pos
106
100
  end
107
101
 
108
- # Decode a single frame
109
- def decode_frame
110
- # Parse frame header
111
- header = Frame::Header.parse(@input_stream)
102
+ # Decode one frame (input positioned after the magic).
103
+ # Returns [frame_output, pos_after_frame].
104
+ def decode_frame(data, pos)
105
+ header, pos = Frame::Header.parse_from(data, pos)
112
106
 
113
- # Calculate window size
114
- calculate_window_size(header)
107
+ # Reset per-frame state: repeat offsets, previous Huffman
108
+ # table, previous FSE tables.
109
+ @executor = SequenceExecutor.new
110
+ @previous_huffman_table = nil
111
+ @previous_fse_tables = {}
115
112
 
116
- # Decode blocks
117
113
  output = String.new(encoding: Encoding::BINARY)
118
-
119
114
  loop do
120
- block = Frame::Block.parse(@input_stream)
121
-
122
- # Decode block content
123
- block_output = decode_block(block, header)
124
- output << block_output
115
+ block, pos = Frame::Block.parse_from(data, pos)
116
+ if block.reserved?
117
+ raise Omnizip::DecompressionError,
118
+ "reserved block type in frame"
119
+ end
125
120
 
121
+ pos = decode_block(block, data, pos, output)
126
122
  break if block.last_block
127
123
  end
128
124
 
129
- # Verify checksum if present
130
125
  if header.content_checksum?
131
- verify_checksum(output)
132
- end
133
-
134
- output
135
- end
126
+ if data.bytesize < pos + 4
127
+ raise Omnizip::DecompressionError, "truncated frame checksum"
128
+ end
136
129
 
137
- # Calculate window size from header
138
- def calculate_window_size(header)
139
- return BLOCK_MAX_SIZE if header.single_segment
140
- return nil unless header.window_log
130
+ expected = data.byteslice(pos, 4).unpack1("V")
131
+ actual = XXHash64.frame_checksum(output)
132
+ if expected != actual
133
+ raise Omnizip::DecompressionError,
134
+ "frame checksum mismatch: stored 0x#{expected.to_s(16)}, " \
135
+ "computed 0x#{actual.to_s(16)}"
136
+ end
137
+ pos += 4
138
+ end
141
139
 
142
- header.window_size || BLOCK_MAX_SIZE
140
+ [output, pos]
143
141
  end
144
142
 
145
- # Decode a single block
146
- def decode_block(block, _header)
143
+ def decode_block(block, data, pos, output)
147
144
  case block.block_type
148
145
  when BLOCK_TYPE_RAW
149
- decode_raw_block(block)
150
- when BLOCK_TYPE_RLE
151
- decode_rle_block(block)
152
- when BLOCK_TYPE_COMPRESSED
153
- decode_compressed_block(block)
154
- else
155
- raise "Reserved block type: #{block.block_type}"
156
- end
157
- end
158
-
159
- # Decode raw (uncompressed) block
160
- def decode_raw_block(block)
161
- @input_stream.read(block.block_size)
162
- end
163
-
164
- # Decode RLE block
165
- def decode_rle_block(block)
166
- byte = @input_stream.read(1)
167
- byte * block.block_size
168
- end
169
-
170
- # Decode compressed block
171
- def decode_compressed_block(_block)
172
- # Record start position for calculating remaining bytes
173
- @input_stream.pos
174
-
175
- # Decode literals section
176
- literals_decoder = LiteralsDecoder.decode(@input_stream,
177
- @previous_huffman_table)
178
- literals = literals_decoder.literals
179
- @previous_huffman_table = literals_decoder.huffman_table
180
-
181
- # Decode sequences section
182
- sequences_decoder = SequencesDecoder.decode(@input_stream,
183
- literals.bytesize,
184
- @previous_fse_tables)
185
- sequences = sequences_decoder.sequences
186
-
187
- # Execute sequences to produce output
188
- if sequences.empty?
189
- # No sequences - literals are the output
190
- literals
191
- else
192
- SequenceExecutor.execute(literals, sequences)
193
- end
194
- end
146
+ if data.bytesize < pos + block.block_size
147
+ raise Omnizip::DecompressionError, "truncated raw block"
148
+ end
195
149
 
196
- # Verify content checksum
197
- def verify_checksum(output)
198
- # Read checksum (4 bytes)
199
- checksum_bytes = @input_stream.read(4)
200
- return unless checksum_bytes && checksum_bytes.length == 4
150
+ output << data.byteslice(pos, block.block_size)
151
+ pos + block.block_size
152
+ when BLOCK_TYPE_RLE
153
+ raise Omnizip::DecompressionError, "truncated RLE block" if data.bytesize < pos + 1
201
154
 
202
- expected = checksum_bytes.unpack1("V")
203
- calculated = xxhash32(output)
155
+ output << (data.getbyte(pos).chr * block.block_size)
156
+ pos + 1
157
+ when BLOCK_TYPE_COMPRESSED
158
+ block_end = pos + block.block_size
159
+ if data.bytesize < block_end
160
+ raise Omnizip::DecompressionError, "truncated compressed block"
161
+ end
204
162
 
205
- if calculated != expected
206
- warn "Zstandard checksum mismatch (expected #{expected}, got #{calculated})"
163
+ decode_compressed_block(data.byteslice(pos...block_end), output)
164
+ block_end
207
165
  end
208
166
  end
209
167
 
210
- # Calculate XXHash32 checksum (simplified)
211
- def xxhash32(data, seed = 0)
212
- # Simplified XXHash32 - for checksum verification only
213
- # Full implementation would use proper XXHash32 algorithm
214
- hash = seed
215
-
216
- data.each_byte do |byte|
217
- hash = ((hash << 5) + hash + byte) & 0xFFFFFFFF
218
- end
219
-
220
- hash
168
+ def decode_compressed_block(block_input, output)
169
+ literals_decoder = LiteralsDecoder.decode(block_input,
170
+ @previous_huffman_table)
171
+ @previous_huffman_table = literals_decoder.huffman_table ||
172
+ @previous_huffman_table
173
+
174
+ sequences_decoder = SequencesDecoder.decode(
175
+ block_input.byteslice(literals_decoder.consumed..),
176
+ @previous_fse_tables,
177
+ @executor,
178
+ )
179
+ # Merge: Repeat-mode tables are not re-emitted, so the entries
180
+ # carried from earlier blocks must survive.
181
+ @previous_fse_tables =
182
+ @previous_fse_tables.merge(sequences_decoder.fse_tables)
183
+
184
+ @executor.execute(literals_decoder.literals,
185
+ sequences_decoder.sequences, output)
221
186
  end
222
187
  end
223
188
  end