omnizip 0.3.8 → 0.3.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop_todo.yml +51 -50
- data/Gemfile +2 -0
- data/Rakefile +3 -0
- data/TODO.refactor/00-overview.md +67 -0
- data/TODO.refactor/01-registry-base-class.md +81 -0
- data/TODO.refactor/02-error-hierarchy.md +57 -0
- data/TODO.refactor/03-autoload-entry-point.md +69 -0
- data/TODO.refactor/04-filter-base-consolidation.md +49 -0
- data/TODO.refactor/05-send-private-methods.md +64 -0
- data/TODO.refactor/06-instance-variable-access.md +69 -0
- data/TODO.refactor/07-respond-to-replacement.md +91 -0
- data/TODO.refactor/08-spec-doubles.md +27 -0
- data/TODO.refactor/09-cli-shared-module.md +35 -0
- data/TODO.refactor/10-format-detector-ocp.md +34 -0
- data/TODO.refactor/11-convenience-decoupling.md +38 -0
- data/TODO.refactor/12-thread-safety.md +8 -0
- data/TODO.refactor/13-lutaml-model-migration.md +82 -0
- data/TODO.refactor/14-add-missing-specs.md +25 -0
- data/lib/omnizip/algorithm.rb +0 -2
- data/lib/omnizip/algorithm_registry.rb +29 -57
- data/lib/omnizip/algorithms/lzma/bit_model.rb +1 -1
- data/lib/omnizip/algorithms/lzma/dictionary.rb +4 -3
- data/lib/omnizip/algorithms/lzma/distance_coder.rb +5 -102
- data/lib/omnizip/algorithms/lzma/length_coder.rb +4 -92
- data/lib/omnizip/algorithms/lzma/literal_decoder.rb +4 -119
- data/lib/omnizip/algorithms/lzma/lzip_decoder.rb +2 -0
- data/lib/omnizip/algorithms/lzma/match_finder.rb +59 -29
- data/lib/omnizip/algorithms/lzma/optimal_encoder.rb +20 -12
- data/lib/omnizip/algorithms/lzma/range_decoder.rb +77 -269
- data/lib/omnizip/algorithms/lzma/range_encoder.rb +0 -9
- data/lib/omnizip/algorithms/lzma/xz_range_encoder_exact.rb +0 -15
- data/lib/omnizip/algorithms/lzma/xz_state.rb +0 -8
- data/lib/omnizip/algorithms/lzma/xz_utils_decoder.rb +189 -968
- data/lib/omnizip/algorithms/lzma2/encoder.rb +0 -2
- data/lib/omnizip/algorithms/lzma2/lzma2_chunk.rb +0 -2
- data/lib/omnizip/algorithms/lzma2/properties.rb +0 -2
- data/lib/omnizip/algorithms/ppmd7/model.rb +1 -1
- data/lib/omnizip/algorithms/ppmd8/context.rb +0 -2
- data/lib/omnizip/algorithms/ppmd8/model.rb +1 -1
- data/lib/omnizip/algorithms/ppmd8.rb +0 -2
- data/lib/omnizip/algorithms/ppmd_base.rb +0 -2
- data/lib/omnizip/algorithms/sevenzip_lzma2.rb +0 -5
- data/lib/omnizip/algorithms/xz_lzma2.rb +0 -5
- data/lib/omnizip/algorithms/zstandard/frame/header.rb +3 -5
- data/lib/omnizip/algorithms.rb +1 -0
- data/lib/omnizip/archive_handler.rb +75 -0
- data/lib/omnizip/archive_handlers/tar_handler.rb +51 -0
- data/lib/omnizip/archive_handlers/zip_handler.rb +73 -0
- data/lib/omnizip/archive_handlers.rb +11 -0
- data/lib/omnizip/checksum_registry.rb +23 -96
- data/lib/omnizip/cli/shared.rb +31 -0
- data/lib/omnizip/cli.rb +6 -43
- data/lib/omnizip/commands/archive_create_command.rb +0 -6
- data/lib/omnizip/commands/archive_extract_command.rb +0 -4
- data/lib/omnizip/commands/archive_list_command.rb +0 -4
- data/lib/omnizip/commands/archive_repair_command.rb +0 -2
- data/lib/omnizip/commands/archive_verify_command.rb +0 -2
- data/lib/omnizip/commands/compress_command.rb +0 -2
- data/lib/omnizip/commands/decompress_command.rb +0 -2
- data/lib/omnizip/commands/list_command.rb +0 -2
- data/lib/omnizip/commands/metadata_command.rb +0 -5
- data/lib/omnizip/commands/parity_create_command.rb +0 -2
- data/lib/omnizip/commands/parity_repair_command.rb +0 -2
- data/lib/omnizip/commands/parity_verify_command.rb +0 -2
- data/lib/omnizip/commands/profile_list_command.rb +0 -2
- data/lib/omnizip/commands/profile_show_command.rb +0 -2
- data/lib/omnizip/convenience.rb +111 -234
- data/lib/omnizip/converter/seven_zip_to_zip_strategy.rb +1 -1
- data/lib/omnizip/crypto/aes256/cipher.rb +6 -1
- data/lib/omnizip/error.rb +22 -18
- data/lib/omnizip/filter.rb +5 -3
- data/lib/omnizip/filter_registry.rb +74 -98
- data/lib/omnizip/filters/bcj2/decoder.rb +196 -130
- data/lib/omnizip/filters/bcj2.rb +0 -2
- data/lib/omnizip/filters/bcj_x86.rb +0 -2
- data/lib/omnizip/filters/filter_base.rb +4 -70
- data/lib/omnizip/filters/registry.rb +0 -2
- data/lib/omnizip/format_detector.rb +21 -12
- data/lib/omnizip/format_registry.rb +79 -38
- data/lib/omnizip/formats/bzip2_file.rb +0 -2
- data/lib/omnizip/formats/cpio/entry.rb +0 -2
- data/lib/omnizip/formats/cpio/reader.rb +0 -1
- data/lib/omnizip/formats/cpio/writer.rb +0 -1
- data/lib/omnizip/formats/cpio.rb +0 -1
- data/lib/omnizip/formats/format_spec_loader.rb +1 -1
- data/lib/omnizip/formats/gzip.rb +0 -2
- data/lib/omnizip/formats/iso/directory_record.rb +6 -3
- data/lib/omnizip/formats/iso/reader.rb +2 -2
- data/lib/omnizip/formats/iso.rb +0 -1
- data/lib/omnizip/formats/lzip.rb +5 -25
- data/lib/omnizip/formats/lzma_alone.rb +7 -25
- data/lib/omnizip/formats/msi/cab_extractor.rb +181 -0
- data/lib/omnizip/formats/msi/constants.rb +157 -0
- data/lib/omnizip/formats/msi/directory_resolver.rb +188 -0
- data/lib/omnizip/formats/msi/entry.rb +128 -0
- data/lib/omnizip/formats/msi/reader.rb +301 -0
- data/lib/omnizip/formats/msi/string_pool.rb +246 -0
- data/lib/omnizip/formats/msi/table_parser.rb +244 -0
- data/lib/omnizip/formats/msi.rb +84 -0
- data/lib/omnizip/formats/ole/allocation_table.rb +10 -1
- data/lib/omnizip/formats/ole/dirent.rb +20 -3
- data/lib/omnizip/formats/ole/header.rb +0 -1
- data/lib/omnizip/formats/ole/ranges_io.rb +21 -3
- data/lib/omnizip/formats/ole/storage.rb +19 -4
- data/lib/omnizip/formats/ole.rb +0 -2
- data/lib/omnizip/formats/rar/block_parser.rb +0 -1
- data/lib/omnizip/formats/rar/compression/ppmd/encoder.rb +1 -1
- data/lib/omnizip/formats/rar/decompressor.rb +38 -1
- data/lib/omnizip/formats/rar/header.rb +0 -1
- data/lib/omnizip/formats/rar/rar5/writer.rb +0 -2
- data/lib/omnizip/formats/rar/reader.rb +2 -3
- data/lib/omnizip/formats/rar/writer.rb +0 -1
- data/lib/omnizip/formats/rar.rb +2 -2
- data/lib/omnizip/formats/rpm/entry.rb +0 -1
- data/lib/omnizip/formats/rpm/header.rb +3 -8
- data/lib/omnizip/formats/rpm/lead.rb +37 -26
- data/lib/omnizip/formats/rpm/tag.rb +0 -1
- data/lib/omnizip/formats/rpm/writer.rb +0 -1
- data/lib/omnizip/formats/seven_zip/bcj2_stream_decompressor.rb +98 -61
- data/lib/omnizip/formats/seven_zip/coder_chain.rb +1 -2
- data/lib/omnizip/formats/seven_zip/encoded_header.rb +0 -1
- data/lib/omnizip/formats/seven_zip/encrypted_header.rb +0 -1
- data/lib/omnizip/formats/seven_zip/header.rb +0 -1
- data/lib/omnizip/formats/seven_zip/header_encryptor.rb +0 -1
- data/lib/omnizip/formats/seven_zip/header_writer.rb +22 -61
- data/lib/omnizip/formats/seven_zip/models/file_entry.rb +1 -1
- data/lib/omnizip/formats/seven_zip/parser.rb +14 -13
- data/lib/omnizip/formats/seven_zip/reader.rb +82 -19
- data/lib/omnizip/formats/seven_zip/split_archive_reader.rb +0 -15
- data/lib/omnizip/formats/seven_zip/split_archive_writer.rb +0 -1
- data/lib/omnizip/formats/seven_zip/writer.rb +22 -68
- data/lib/omnizip/formats/seven_zip.rb +63 -5
- data/lib/omnizip/formats/tar/entry.rb +0 -2
- data/lib/omnizip/formats/tar/header.rb +0 -2
- data/lib/omnizip/formats/tar/reader.rb +0 -2
- data/lib/omnizip/formats/tar/writer.rb +0 -2
- data/lib/omnizip/formats/tar.rb +0 -1
- data/lib/omnizip/formats/xar/entry.rb +0 -1
- data/lib/omnizip/formats/xar/header.rb +0 -1
- data/lib/omnizip/formats/xar/reader.rb +0 -1
- data/lib/omnizip/formats/xar/toc.rb +3 -3
- data/lib/omnizip/formats/xar/writer.rb +11 -3
- data/lib/omnizip/formats/xar.rb +0 -1
- data/lib/omnizip/formats/xz/reader.rb +7 -3
- data/lib/omnizip/formats/xz.rb +3 -40
- data/lib/omnizip/formats/xz_impl/block_decoder.rb +4 -10
- data/lib/omnizip/formats/xz_impl/block_encoder.rb +0 -2
- data/lib/omnizip/formats/xz_impl/block_header_parser.rb +0 -2
- data/lib/omnizip/formats/xz_impl/index_decoder.rb +0 -1
- data/lib/omnizip/formats/xz_impl/index_encoder.rb +0 -1
- data/lib/omnizip/formats/xz_impl/stream_encoder.rb +1 -1
- data/lib/omnizip/formats/xz_impl/stream_footer.rb +0 -1
- data/lib/omnizip/formats/xz_impl/stream_footer_parser.rb +0 -1
- data/lib/omnizip/formats/xz_impl/stream_header.rb +0 -1
- data/lib/omnizip/formats/xz_impl/stream_header_parser.rb +0 -2
- data/lib/omnizip/formats/zip/central_directory_header.rb +2 -3
- data/lib/omnizip/formats/zip/end_of_central_directory.rb +0 -2
- data/lib/omnizip/formats/zip/local_file_header.rb +0 -2
- data/lib/omnizip/formats/zip/reader.rb +12 -2
- data/lib/omnizip/formats/zip/writer.rb +73 -1
- data/lib/omnizip/formats/zip/zip64_end_of_central_directory.rb +0 -2
- data/lib/omnizip/formats/zip/zip64_end_of_central_directory_locator.rb +0 -2
- data/lib/omnizip/formats/zip/zip64_extra_field.rb +0 -2
- data/lib/omnizip/formats/zip.rb +0 -1
- data/lib/omnizip/formats.rb +9 -22
- data/lib/omnizip/implementations/seven_zip/lzma/range_decoder.rb +2 -2
- data/lib/omnizip/implementations/seven_zip/lzma/range_encoder.rb +1 -1
- data/lib/omnizip/implementations/seven_zip/lzma2/encoder.rb +0 -6
- data/lib/omnizip/implementations/xz_utils/lzma2/decoder.rb +18 -185
- data/lib/omnizip/implementations/xz_utils/lzma2/encoder.rb +3 -3
- data/lib/omnizip/io/source.rb +136 -0
- data/lib/omnizip/io.rb +2 -0
- data/lib/omnizip/link_handler.rb +7 -3
- data/lib/omnizip/metadata/metadata_editor.rb +1 -1
- data/lib/omnizip/models/algorithm_metadata.rb +33 -60
- data/lib/omnizip/models/compression_options.rb +40 -53
- data/lib/omnizip/models/conversion_options.rb +9 -18
- data/lib/omnizip/models/parallel_options.rb +25 -0
- data/lib/omnizip/optimization_registry.rb +8 -44
- data/lib/omnizip/parallel/job_scheduler.rb +26 -8
- data/lib/omnizip/parallel/parallel_compressor.rb +10 -20
- data/lib/omnizip/parallel/parallel_extractor.rb +20 -11
- data/lib/omnizip/parity/par2_repairer.rb +5 -5
- data/lib/omnizip/parity/par2_verifier.rb +27 -22
- data/lib/omnizip/parity.rb +4 -4
- data/lib/omnizip/password/encryption_registry.rb +11 -50
- data/lib/omnizip/password/zip_crypto_strategy.rb +1 -4
- data/lib/omnizip/pipe/stream_decompressor.rb +2 -3
- data/lib/omnizip/profile/archive_profile.rb +0 -2
- data/lib/omnizip/profile/binary_profile.rb +0 -2
- data/lib/omnizip/profile/compression_profile.rb +0 -1
- data/lib/omnizip/profile/profile_detector.rb +0 -2
- data/lib/omnizip/profile/text_profile.rb +0 -2
- data/lib/omnizip/profile.rb +0 -2
- data/lib/omnizip/registry.rb +112 -0
- data/lib/omnizip/version.rb +1 -1
- data/lib/omnizip/zip/file.rb +20 -9
- data/lib/omnizip.rb +26 -55
- data/lib/tasks/msi_test_files.rake +66 -0
- metadata +57 -6
|
@@ -49,6 +49,8 @@ module Omnizip
|
|
|
49
49
|
end
|
|
50
50
|
|
|
51
51
|
# Position encoders for slots 4-13
|
|
52
|
+
# SDK convention: size = NUM_FULL_DISTANCES - END_POS_MODEL_INDEX = 128 - 14 = 114
|
|
53
|
+
# Indexed by: base - slot - 1 (matches LZMA SDK and XZ Utils)
|
|
52
54
|
@pos_encoders = Array.new(NUM_FULL_DISTANCES - END_POS_MODEL_INDEX) do
|
|
53
55
|
BitModel.new
|
|
54
56
|
end
|
|
@@ -71,19 +73,11 @@ module Omnizip
|
|
|
71
73
|
#
|
|
72
74
|
# @return [void]
|
|
73
75
|
def reset_models
|
|
74
|
-
if ENV.fetch("DEBUG_RESET_MODELS",
|
|
75
|
-
nil) && ENV.fetch("LZMA_DEBUG_DISTANCE", nil)
|
|
76
|
-
puts " [DistanceCoder.reset_models] Resetting #{@slot_encoders.size} len_states, each with #{@slot_encoders[0]&.size || '?'} models"
|
|
77
|
-
end
|
|
78
76
|
@slot_encoders.each do |len_state_models|
|
|
79
77
|
len_state_models.each(&:reset)
|
|
80
78
|
end
|
|
81
79
|
@pos_encoders.each(&:reset)
|
|
82
80
|
@align_encoder.each(&:reset)
|
|
83
|
-
if ENV.fetch("DEBUG_RESET_MODELS",
|
|
84
|
-
nil) && ENV.fetch("LZMA_DEBUG_DISTANCE", nil)
|
|
85
|
-
puts " [DistanceCoder.reset_models] Done resetting"
|
|
86
|
-
end
|
|
87
81
|
end
|
|
88
82
|
|
|
89
83
|
# Encode a match distance using SDK-compatible encoding
|
|
@@ -95,11 +89,6 @@ module Omnizip
|
|
|
95
89
|
def encode(range_encoder, distance, len_state)
|
|
96
90
|
slot = get_dist_slot(distance)
|
|
97
91
|
|
|
98
|
-
if ENV["LZMA_DEBUG_ENCODE"]
|
|
99
|
-
puts "[DistanceCoder.encode] distance=#{distance} slot=#{slot} len_state=#{len_state}"
|
|
100
|
-
puts "[DistanceCoder.encode] CALLING encode_tree with symbol=#{slot}"
|
|
101
|
-
end
|
|
102
|
-
|
|
103
92
|
# Encode the slot using the appropriate slot encoder
|
|
104
93
|
encode_tree(range_encoder, @slot_encoders[len_state], slot,
|
|
105
94
|
NUM_DIST_SLOT_BITS)
|
|
@@ -111,6 +100,7 @@ module Omnizip
|
|
|
111
100
|
|
|
112
101
|
if slot < END_POS_MODEL_INDEX
|
|
113
102
|
# Slots 4-13: Use position encoders (reverse tree encoding)
|
|
103
|
+
# SDK/XZ Utils convention: base_idx = base - slot - 1
|
|
114
104
|
encode_reverse_tree(range_encoder,
|
|
115
105
|
@pos_encoders,
|
|
116
106
|
base - slot - 1,
|
|
@@ -138,61 +128,24 @@ module Omnizip
|
|
|
138
128
|
# @param len_state [Integer] Length state for slot selection
|
|
139
129
|
# @return [Integer] Decoded distance value (before adding 1)
|
|
140
130
|
def decode(range_decoder, len_state)
|
|
141
|
-
# DEBUG: Trace specific calls to find corruption
|
|
142
|
-
$distance_decode_count ||= 0
|
|
143
|
-
debug_calls = (320..330)
|
|
144
|
-
debug_this = debug_calls.include?($distance_decode_count)
|
|
145
|
-
trace_326 = ($distance_decode_count == 326)
|
|
146
|
-
trace_325 = ($distance_decode_count == 325)
|
|
147
|
-
|
|
148
|
-
# DEBUG: Trace large distances (> 100000)
|
|
149
|
-
trace_large = $distance_decode_count.between?(25,
|
|
150
|
-
35) || $distance_decode_count.between?(
|
|
151
|
-
315, 330
|
|
152
|
-
)
|
|
153
|
-
|
|
154
|
-
# DEBUG: Trace all when LZMA_DEBUG_DISTANCE is set
|
|
155
|
-
trace_all = ENV.fetch("LZMA_DEBUG_DISTANCE", nil)
|
|
156
|
-
|
|
157
|
-
if (trace_325 || trace_large || trace_all) && ENV.fetch(
|
|
158
|
-
"LZMA_DEBUG_DISTANCE", nil
|
|
159
|
-
)
|
|
160
|
-
puts " [DistanceCoder.decode ##{$distance_decode_count}] START - len_state=#{len_state}"
|
|
161
|
-
puts " BEFORE: range=#{range_decoder.range.inspect}, code=#{range_decoder.code.inspect}"
|
|
162
|
-
end
|
|
163
|
-
|
|
164
131
|
slot = decode_tree(range_decoder, @slot_encoders[len_state],
|
|
165
132
|
NUM_DIST_SLOT_BITS)
|
|
166
133
|
|
|
167
|
-
if (debug_this || trace_large || trace_all) && ENV.fetch(
|
|
168
|
-
"LZMA_DEBUG_DISTANCE", nil
|
|
169
|
-
)
|
|
170
|
-
puts " [DistanceCoder.decode ##{$distance_decode_count}] len_state=#{len_state}, slot=#{slot}"
|
|
171
|
-
puts " @slot_encoders[#{len_state}] object_id=#{@slot_encoders[len_state].object_id}"
|
|
172
|
-
end
|
|
173
|
-
|
|
174
134
|
# Decode extra bits based on slot
|
|
175
135
|
if slot < START_POS_MODEL_INDEX
|
|
176
136
|
# Slots 0-3: No extra bits
|
|
177
|
-
$distance_decode_count += 1
|
|
178
|
-
if debug_this && ENV.fetch("LZMA_DEBUG_DISTANCE", nil)
|
|
179
|
-
puts " -> distance=#{slot}"
|
|
180
|
-
end
|
|
181
137
|
slot
|
|
182
138
|
else
|
|
183
139
|
footer_bits = (slot >> 1) - 1
|
|
184
140
|
|
|
185
141
|
if slot < END_POS_MODEL_INDEX
|
|
186
142
|
# Slots 4-13: Use position encoders (reverse tree decoding)
|
|
143
|
+
# SDK/XZ Utils convention: base_idx = base - slot - 1
|
|
187
144
|
base = (2 | (slot & 1)) << footer_bits
|
|
188
145
|
result = base + decode_reverse_tree(range_decoder,
|
|
189
146
|
@pos_encoders,
|
|
190
147
|
base - slot - 1,
|
|
191
148
|
footer_bits)
|
|
192
|
-
$distance_decode_count += 1
|
|
193
|
-
if debug_this && ENV.fetch("LZMA_DEBUG_DISTANCE", nil)
|
|
194
|
-
puts " -> distance=#{result} (slot #{slot})"
|
|
195
|
-
end
|
|
196
149
|
else
|
|
197
150
|
# Slots 14+: Fixed direct bits + aligned bits
|
|
198
151
|
# XZ Utils pattern (lzma_decoder.c:500-514):
|
|
@@ -225,23 +178,10 @@ module Omnizip
|
|
|
225
178
|
@align_encoder,
|
|
226
179
|
0,
|
|
227
180
|
DIST_ALIGN_BITS)
|
|
228
|
-
if trace_326 && ENV.fetch("LZMA_DEBUG_DISTANCE", nil)
|
|
229
|
-
puts " TRACE_326: low_bits=#{low_bits}"
|
|
230
|
-
end
|
|
231
181
|
|
|
232
182
|
# Final result: (result << 4) + low_bits
|
|
233
183
|
# NOTE: slot value is NOT added (XZ Utils pattern - line 513 adds symbol for EOPM check only)
|
|
234
184
|
result = (result << DIST_ALIGN_BITS) + low_bits
|
|
235
|
-
$distance_decode_count += 1
|
|
236
|
-
if (debug_this || trace_large) && ENV.fetch(
|
|
237
|
-
"LZMA_DEBUG_DISTANCE", nil
|
|
238
|
-
)
|
|
239
|
-
puts " -> slot=#{slot}, result_after_direct=#{result >> DIST_ALIGN_BITS}, low_bits=#{low_bits}, distance=#{result}"
|
|
240
|
-
end
|
|
241
|
-
if result > 100000
|
|
242
|
-
puts " [LARGE_DISTANCE ##{$distance_decode_count}] distance=#{result}, slot=#{slot}" if ENV["LZMA_DEBUG_DISTANCE"]
|
|
243
|
-
puts " BEFORE: range_decoder.range=#{range_decoder.range}, range_decoder.code=#{range_decoder.code}" if ENV["LZMA_DEBUG_DISTANCE"]
|
|
244
|
-
end
|
|
245
185
|
end
|
|
246
186
|
result
|
|
247
187
|
end
|
|
@@ -321,29 +261,11 @@ module Omnizip
|
|
|
321
261
|
# @return [void]
|
|
322
262
|
def encode_tree(range_encoder, models, symbol, num_bits)
|
|
323
263
|
m = 1
|
|
324
|
-
trace_all = ENV.fetch("TRACE_ALL_SLOT_ENCODE", nil)
|
|
325
|
-
iteration = 0
|
|
326
|
-
|
|
327
|
-
if trace_all && ENV.fetch("LZMA_DEBUG_ENCODE", nil)
|
|
328
|
-
puts " [encode_tree START] RECEIVED symbol=#{symbol}, num_bits=#{num_bits}"
|
|
329
|
-
puts " BEFORE: range=#{range_encoder.range}, low=#{range_encoder.low}"
|
|
330
|
-
end
|
|
331
|
-
|
|
332
264
|
(num_bits - 1).downto(0) do |i|
|
|
333
|
-
iteration += 1
|
|
334
265
|
bit = (symbol >> i) & 1
|
|
335
|
-
if trace_all && ENV.fetch("LZMA_DEBUG_ENCODE", nil)
|
|
336
|
-
model_idx = m
|
|
337
|
-
puts " [#{iteration}/#{num_bits}] i=#{i}, bit=#{bit}, m=#{m}, model_idx=#{model_idx}, prob=#{models[m].probability}"
|
|
338
|
-
end
|
|
339
266
|
range_encoder.encode_bit(models[m], bit)
|
|
340
267
|
m = (m << 1) | bit
|
|
341
268
|
end
|
|
342
|
-
|
|
343
|
-
if trace_all && ENV.fetch("LZMA_DEBUG_ENCODE", nil)
|
|
344
|
-
puts " AFTER: range=#{range_encoder.range}, low=#{range_encoder.low}"
|
|
345
|
-
puts " [encode_tree END] ENCODED symbol=#{symbol}"
|
|
346
|
-
end
|
|
347
269
|
end
|
|
348
270
|
|
|
349
271
|
# Decode a value using a tree of bit models
|
|
@@ -355,29 +277,10 @@ module Omnizip
|
|
|
355
277
|
def decode_tree(range_decoder, models, num_bits)
|
|
356
278
|
m = 1
|
|
357
279
|
symbol = 0
|
|
358
|
-
trace_this = (num_bits == 6 && ENV.fetch("TRACE_SLOT_DECODE",
|
|
359
|
-
nil)) || ($distance_decode_count == 28)
|
|
360
|
-
trace_all = ENV.fetch("TRACE_ALL_SLOT_DECODE", nil)
|
|
361
|
-
iteration = 0
|
|
362
|
-
|
|
363
|
-
if (trace_this || trace_all) && ENV.fetch("LZMA_DEBUG_DISTANCE", nil)
|
|
364
|
-
puts " [decode_tree START] num_bits=#{num_bits}, range=#{range_decoder.range}, code=#{range_decoder.code}"
|
|
365
|
-
puts " models array object_id=#{models.object_id}"
|
|
366
|
-
end
|
|
367
|
-
|
|
368
280
|
(num_bits - 1).downto(0) do |i|
|
|
369
|
-
|
|
370
|
-
model = models[m]
|
|
371
|
-
bit = range_decoder.decode_bit(model)
|
|
281
|
+
bit = range_decoder.decode_bit(models[m])
|
|
372
282
|
m = (m << 1) | bit
|
|
373
283
|
symbol |= (bit << i)
|
|
374
|
-
if (trace_this || trace_all) && ENV.fetch("LZMA_DEBUG_DISTANCE",
|
|
375
|
-
nil)
|
|
376
|
-
puts " [#{iteration}/#{num_bits}] i=#{i}, bit=#{bit}, m=#{m}, model.object_id=#{model.object_id}, prob=#{model.probability}, symbol=#{symbol}"
|
|
377
|
-
end
|
|
378
|
-
end
|
|
379
|
-
if (trace_this || trace_all) && ENV.fetch("LZMA_DEBUG_DISTANCE", nil)
|
|
380
|
-
puts " [decode_tree END] symbol=#{symbol}"
|
|
381
284
|
end
|
|
382
285
|
symbol
|
|
383
286
|
end
|
|
@@ -67,67 +67,25 @@ module Omnizip
|
|
|
67
67
|
# @param pos_state [Integer] Position state for tree selection
|
|
68
68
|
# @return [void]
|
|
69
69
|
def encode(range_encoder, length, pos_state)
|
|
70
|
-
trace_encode = ENV.fetch("LZMA_DEBUG_ENCODE",
|
|
71
|
-
nil) && ENV.fetch("TRACE_LENGTH_CODER", nil)
|
|
72
|
-
|
|
73
|
-
if trace_encode
|
|
74
|
-
puts " [LengthCoder.encode] START: length=#{length}, pos_state=#{pos_state}"
|
|
75
|
-
puts " @choice.prob=#{@choice.probability} @choice2.prob=#{@choice2.probability}"
|
|
76
|
-
end
|
|
77
|
-
|
|
78
70
|
if length < LEN_LOW_SYMBOLS
|
|
79
71
|
# 0-7: Use low tree
|
|
80
|
-
if trace_encode
|
|
81
|
-
puts " Using LOW tree (length #{length} < #{LEN_LOW_SYMBOLS})"
|
|
82
|
-
puts " Encoding choice=0 with prob=#{@choice.probability}"
|
|
83
|
-
end
|
|
84
72
|
range_encoder.encode_bit(@choice, 0)
|
|
85
|
-
if trace_encode
|
|
86
|
-
puts " After choice: @choice.prob=#{@choice.probability}"
|
|
87
|
-
end
|
|
88
73
|
encode_tree(range_encoder, @low[pos_state], length,
|
|
89
74
|
NUM_LEN_LOW_BITS)
|
|
90
75
|
elsif length < LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS
|
|
91
76
|
# 8-15: Use mid tree
|
|
92
|
-
if trace_encode
|
|
93
|
-
puts " Using MID tree (length #{length} < #{LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS})"
|
|
94
|
-
puts " Encoding choice=1 with prob=#{@choice.probability}"
|
|
95
|
-
end
|
|
96
77
|
range_encoder.encode_bit(@choice, 1)
|
|
97
|
-
if trace_encode
|
|
98
|
-
puts " After choice: @choice.prob=#{@choice.probability}"
|
|
99
|
-
puts " Encoding choice2=0 with prob=#{@choice2.probability}"
|
|
100
|
-
end
|
|
101
78
|
range_encoder.encode_bit(@choice2, 0)
|
|
102
|
-
if trace_encode
|
|
103
|
-
puts " After choice2: @choice2.prob=#{@choice2.probability}"
|
|
104
|
-
end
|
|
105
79
|
encode_tree(range_encoder, @mid[pos_state],
|
|
106
80
|
length - LEN_LOW_SYMBOLS, NUM_LEN_MID_BITS)
|
|
107
81
|
else
|
|
108
82
|
# 16+: Use high tree
|
|
109
|
-
if trace_encode
|
|
110
|
-
puts " Using HIGH tree (length #{length} >= #{LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS})"
|
|
111
|
-
puts " Encoding choice=1 with prob=#{@choice.probability}"
|
|
112
|
-
end
|
|
113
83
|
range_encoder.encode_bit(@choice, 1)
|
|
114
|
-
if trace_encode
|
|
115
|
-
puts " After choice: @choice.prob=#{@choice.probability}"
|
|
116
|
-
puts " Encoding choice2=1 with prob=#{@choice2.probability}"
|
|
117
|
-
end
|
|
118
84
|
range_encoder.encode_bit(@choice2, 1)
|
|
119
|
-
if trace_encode
|
|
120
|
-
puts " After choice2: @choice2.prob=#{@choice2.probability}"
|
|
121
|
-
end
|
|
122
85
|
encode_tree(range_encoder, @high,
|
|
123
86
|
length - LEN_LOW_SYMBOLS - LEN_MID_SYMBOLS,
|
|
124
87
|
NUM_LEN_HIGH_BITS)
|
|
125
88
|
end
|
|
126
|
-
|
|
127
|
-
if trace_encode
|
|
128
|
-
puts " FINAL @choice.prob=#{@choice.probability} @choice2.prob=#{@choice2.probability}"
|
|
129
|
-
puts " [LengthCoder.encode] END"
|
|
130
|
-
end
|
|
131
89
|
end
|
|
132
90
|
|
|
133
91
|
# Decode a match length using SDK-compatible decoding
|
|
@@ -136,58 +94,21 @@ module Omnizip
|
|
|
136
94
|
# @param pos_state [Integer] Position state for tree selection
|
|
137
95
|
# @return [Integer] Decoded length value (before adding MATCH_LEN_MIN)
|
|
138
96
|
def decode(range_decoder, pos_state)
|
|
139
|
-
trace_decode = ENV.fetch("LZMA_DEBUG_DISTANCE",
|
|
140
|
-
nil) && ENV.fetch("TRACE_LENGTH_CODER", nil)
|
|
141
|
-
|
|
142
|
-
if trace_decode
|
|
143
|
-
caller_loc = caller_locations(2, 1).first
|
|
144
|
-
puts " [LengthCoder.decode] START: pos_state=#{pos_state}"
|
|
145
|
-
puts " self.object_id=#{object_id}"
|
|
146
|
-
puts " @choice.object_id=#{@choice.object_id} prob=#{@choice.probability}"
|
|
147
|
-
puts " @choice2.object_id=#{@choice2.object_id} prob=#{@choice2.probability}"
|
|
148
|
-
puts " Called from: #{caller_loc.label} at #{caller_loc.lineno}"
|
|
149
|
-
end
|
|
150
|
-
|
|
151
97
|
choice_bit = range_decoder.decode_bit(@choice)
|
|
152
|
-
if trace_decode
|
|
153
|
-
puts " Decoded choice=#{choice_bit} with prob=#{@choice.probability}"
|
|
154
|
-
puts " After choice decode: @choice.prob=#{@choice.probability}"
|
|
155
|
-
end
|
|
156
98
|
|
|
157
99
|
if choice_bit.zero?
|
|
158
100
|
# Low tree
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
end
|
|
162
|
-
result = decode_tree(range_decoder, @low[pos_state],
|
|
163
|
-
NUM_LEN_LOW_BITS)
|
|
101
|
+
decode_tree(range_decoder, @low[pos_state],
|
|
102
|
+
NUM_LEN_LOW_BITS)
|
|
164
103
|
elsif range_decoder.decode_bit(@choice2).zero?
|
|
165
104
|
# Mid tree
|
|
166
|
-
|
|
167
|
-
puts " Decoded choice2=0 with prob=#{@choice2.probability}"
|
|
168
|
-
puts " After choice2 decode: @choice2.prob=#{@choice2.probability}"
|
|
169
|
-
puts " Using MID tree"
|
|
170
|
-
end
|
|
171
|
-
result = LEN_LOW_SYMBOLS +
|
|
105
|
+
LEN_LOW_SYMBOLS +
|
|
172
106
|
decode_tree(range_decoder, @mid[pos_state], NUM_LEN_MID_BITS)
|
|
173
107
|
else
|
|
174
108
|
# High tree
|
|
175
|
-
|
|
176
|
-
puts " Decoded choice2=1 with prob=#{@choice2.probability}"
|
|
177
|
-
puts " After choice2 decode: @choice2.prob=#{@choice2.probability}"
|
|
178
|
-
puts " Using HIGH tree"
|
|
179
|
-
end
|
|
180
|
-
result = LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS +
|
|
109
|
+
LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS +
|
|
181
110
|
decode_tree(range_decoder, @high, NUM_LEN_HIGH_BITS)
|
|
182
111
|
end
|
|
183
|
-
|
|
184
|
-
if trace_decode
|
|
185
|
-
puts " FINAL @choice.prob=#{@choice.probability} @choice2.prob=#{@choice2.probability}"
|
|
186
|
-
puts " Result: length_encoded=#{result}"
|
|
187
|
-
puts " [LengthCoder.decode] END"
|
|
188
|
-
end
|
|
189
|
-
|
|
190
|
-
result
|
|
191
112
|
end
|
|
192
113
|
|
|
193
114
|
# Reset probability models to initial values
|
|
@@ -197,12 +118,6 @@ module Omnizip
|
|
|
197
118
|
#
|
|
198
119
|
# @return [void]
|
|
199
120
|
def reset_models
|
|
200
|
-
if ENV["TRACE_RESET_MODELS"]
|
|
201
|
-
puts " [LengthCoder.reset_models] CALLED!"
|
|
202
|
-
puts " Before reset: @choice.prob=#{@choice.probability} @choice2.prob=#{@choice2.probability}"
|
|
203
|
-
caller_loc = caller_locations(2, 1).first
|
|
204
|
-
puts " Called from: #{caller_loc.label} at #{caller_loc.path}:#{caller_loc.lineno}"
|
|
205
|
-
end
|
|
206
121
|
@choice.reset
|
|
207
122
|
@choice2.reset
|
|
208
123
|
|
|
@@ -215,9 +130,6 @@ module Omnizip
|
|
|
215
130
|
end
|
|
216
131
|
|
|
217
132
|
@high.each(&:reset)
|
|
218
|
-
if ENV["TRACE_RESET_MODELS"]
|
|
219
|
-
puts " After reset: @choice.prob=#{@choice.probability} @choice2.prob=#{@choice2.probability}"
|
|
220
|
-
end
|
|
221
133
|
end
|
|
222
134
|
|
|
223
135
|
private
|
|
@@ -68,24 +68,6 @@ module Omnizip
|
|
|
68
68
|
# We build it up bit by bit until it reaches 0x100
|
|
69
69
|
symbol = 1
|
|
70
70
|
|
|
71
|
-
# TEMP DEBUG: Trace first literal decode
|
|
72
|
-
if ENV["TRACE_LITERAL_DECODE"] && lit_state.zero?
|
|
73
|
-
# range = range_decoder.instance_variable_get(:@range)
|
|
74
|
-
# code = range_decoder.instance_variable_get(:@code)
|
|
75
|
-
# puts ""
|
|
76
|
-
# puts "=== decode_unmatched START: lit_state=#{lit_state}, base_offset=#{base_offset} ==="
|
|
77
|
-
# puts "Initial: range=0x#{range.to_s(16)}, code=0x#{code.to_s(16)}"
|
|
78
|
-
end
|
|
79
|
-
|
|
80
|
-
# DEBUG: Trace lit_state=96 (the corrupted literal)
|
|
81
|
-
if lit_state == 96
|
|
82
|
-
# range = range_decoder.instance_variable_get(:@range)
|
|
83
|
-
# code = range_decoder.instance_variable_get(:@code)
|
|
84
|
-
# puts ""
|
|
85
|
-
# puts "=== decode_unmatched START: lit_state=#{lit_state}, base_offset=#{base_offset} ==="
|
|
86
|
-
# puts "Initial: range=0x#{range.to_s(16)}, code=0x#{code.to_s(16)}"
|
|
87
|
-
end
|
|
88
|
-
|
|
89
71
|
# Decode 8 bits to build the symbol from 1 to 0x100
|
|
90
72
|
while symbol < 0x100
|
|
91
73
|
# Model index based on current symbol value
|
|
@@ -94,34 +76,13 @@ module Omnizip
|
|
|
94
76
|
# Decode next bit
|
|
95
77
|
bit = range_decoder.decode_bit(models[model_index])
|
|
96
78
|
|
|
97
|
-
if ENV["TRACE_LITERAL_DECODE"] && lit_state.zero?
|
|
98
|
-
range_after = range_decoder.instance_variable_get(:@range)
|
|
99
|
-
code_after = range_decoder.instance_variable_get(:@code)
|
|
100
|
-
puts "Bit #{symbol}: model_index=#{model_index}, bit=#{bit}, range=0x#{range_after.to_s(16)}, code=0x#{code_after.to_s(16)}" if ENV["LZMA_DEBUG_BITS"]
|
|
101
|
-
end
|
|
102
|
-
|
|
103
|
-
# DEBUG: Trace bits for lit_state=96
|
|
104
|
-
if ENV["LZMA_DEBUG_BITS"] && lit_state == 96
|
|
105
|
-
range_after = range_decoder.instance_variable_get(:@range)
|
|
106
|
-
code_after = range_decoder.instance_variable_get(:@code)
|
|
107
|
-
puts " symbol=#{symbol}: model_index=#{model_index}, bit=#{bit}, range=0x#{range_after.to_s(16)}, code=0x#{code_after.to_s(16)}"
|
|
108
|
-
end
|
|
109
|
-
|
|
110
79
|
# Update symbol: shift left and add bit
|
|
111
80
|
symbol = (symbol << 1) | bit
|
|
112
81
|
end
|
|
113
82
|
|
|
114
83
|
# Symbol is now in range 0x100-0x1FF
|
|
115
84
|
# Extract the byte value by subtracting 0x100
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
if ENV["TRACE_LITERAL_DECODE"] && lit_state.zero?
|
|
119
|
-
puts "Result: 0x#{result.to_s(16)} ('#{result.chr}')"
|
|
120
|
-
puts "=== decode_unmatched END ==="
|
|
121
|
-
puts ""
|
|
122
|
-
end
|
|
123
|
-
|
|
124
|
-
result
|
|
85
|
+
symbol - 0x100
|
|
125
86
|
end
|
|
126
87
|
|
|
127
88
|
# Decode literal byte in matched mode (SDK feature)
|
|
@@ -153,34 +114,9 @@ module Omnizip
|
|
|
153
114
|
# See: /Users/mulgogi/src/external/xz/src/liblzma/rangecoder/range_decoder.h:342-357
|
|
154
115
|
offset = 0x100
|
|
155
116
|
|
|
156
|
-
# DEBUG: Trace matched literal decode at position 61
|
|
157
|
-
if ENV["TRACE_MATCHED_DECODE"] && lit_state == 96
|
|
158
|
-
puts "=== MATCHED LITERAL DECODE: lit_state=#{lit_state}, match_byte=0x#{match_byte.to_s(16).upcase} ==="
|
|
159
|
-
puts " base_offset=#{base_offset}"
|
|
160
|
-
puts " Initial: symbol=#{symbol}, offset=0x#{offset.to_s(16).upcase}"
|
|
161
|
-
end
|
|
162
|
-
|
|
163
117
|
# SDK matched literal decoding algorithm
|
|
164
118
|
# Process bits while match byte provides context
|
|
165
|
-
bit_num = 0
|
|
166
|
-
result_bits = [] # DEBUG: Track decoded bits
|
|
167
|
-
|
|
168
|
-
# DEBUG: Trace at dict_full=233
|
|
169
|
-
trace_233 = ENV.fetch("DICT_FULL_233_TRACE", nil) && lit_state.zero?
|
|
170
|
-
|
|
171
|
-
if trace_233
|
|
172
|
-
puts "=== MATCHED LITERAL TRACE at dict_full=233 ==="
|
|
173
|
-
puts " match_byte=0x#{match_byte.to_s(16).upcase}"
|
|
174
|
-
puts " base_offset=#{base_offset}"
|
|
175
|
-
puts " Initial: symbol=#{symbol}, offset=0x#{offset.to_s(16).upcase}"
|
|
176
|
-
end
|
|
177
|
-
|
|
178
119
|
loop do
|
|
179
|
-
if trace_233
|
|
180
|
-
puts "\n Bit #{bit_num}:"
|
|
181
|
-
puts " match_sym=0x#{(match_sym & 0xFF).to_s(16).upcase}, offset=0x#{offset.to_s(16).upcase}"
|
|
182
|
-
end
|
|
183
|
-
|
|
184
120
|
# XZ Utils pattern: t_match_byte <<= 1; t_match_bit = t_match_byte & t_offset;
|
|
185
121
|
# IMPORTANT: Shift FIRST, then extract the bit!
|
|
186
122
|
# Shift match symbol (brings next bit into position 8)
|
|
@@ -198,31 +134,8 @@ module Omnizip
|
|
|
198
134
|
# where t_offset is updated based on PREVIOUS decoded bit, t_match_bit is from match byte
|
|
199
135
|
model_index = base_offset + offset + match_bit + symbol
|
|
200
136
|
|
|
201
|
-
if trace_233
|
|
202
|
-
puts " match_bit=0x#{match_bit.to_s(16).upcase}, symbol=#{symbol}"
|
|
203
|
-
puts " model_index=#{model_index}"
|
|
204
|
-
puts " offset_from_base=#{model_index - base_offset}"
|
|
205
|
-
prob_before = models[model_index].probability
|
|
206
|
-
puts " probability_before=0x#{prob_before.to_s(16).upcase} (#{prob_before})"
|
|
207
|
-
# Trace range decoder state BEFORE decode_bit
|
|
208
|
-
rd_range_before = range_decoder.instance_variable_get(:@range)
|
|
209
|
-
rd_code_before = range_decoder.instance_variable_get(:@code)
|
|
210
|
-
puts " range_decoder BEFORE: range=0x#{rd_range_before.to_s(16)}, code=0x#{rd_code_before.to_s(16)}"
|
|
211
|
-
end
|
|
212
|
-
|
|
213
137
|
# Decode literal bit
|
|
214
138
|
bit = range_decoder.decode_bit(models[model_index])
|
|
215
|
-
result_bits << bit # DEBUG: Track bit
|
|
216
|
-
|
|
217
|
-
if trace_233
|
|
218
|
-
prob_after = models[model_index].probability
|
|
219
|
-
puts " decoded_bit=#{bit}"
|
|
220
|
-
puts " probability_after=0x#{prob_after.to_s(16).upcase} (#{prob_after})"
|
|
221
|
-
# Also trace the range decoder state after decode_bit
|
|
222
|
-
rd_range = range_decoder.instance_variable_get(:@range)
|
|
223
|
-
rd_code = range_decoder.instance_variable_get(:@code)
|
|
224
|
-
puts " range_decoder AFTER: range=0x#{rd_range.to_s(16)}, code=0x#{rd_code.to_s(16)}"
|
|
225
|
-
end
|
|
226
139
|
|
|
227
140
|
# Update offset based on DECODED bit (XZ Utils pattern)
|
|
228
141
|
# IMPORTANT: XZ Utils rc_bit macro updates symbol BEFORE running the action!
|
|
@@ -246,48 +159,20 @@ module Omnizip
|
|
|
246
159
|
symbol = (symbol << 1) | 1
|
|
247
160
|
end
|
|
248
161
|
|
|
249
|
-
if ENV["TRACE_MATCHED_DECODE"] && lit_state == 96
|
|
250
|
-
puts " new_offset=0x#{offset.to_s(16).upcase}"
|
|
251
|
-
puts " new_symbol=#{symbol} (0x#{symbol.to_s(16).upcase})"
|
|
252
|
-
end
|
|
253
|
-
|
|
254
162
|
# If bits diverge, switch to unmatched mode
|
|
255
163
|
if (match_bit.positive? ? 1 : 0) != bit
|
|
256
|
-
if ENV["TRACE_MATCHED_DECODE"] && lit_state == 96
|
|
257
|
-
puts " *** BITS DIVERGE - switching to unmatched mode ***"
|
|
258
|
-
end
|
|
259
|
-
if trace_233
|
|
260
|
-
puts " *** BITS DIVERGE at bit #{bit_num} - match_bit=#{match_bit.positive? ? 1 : 0}, decoded_bit=#{bit} ***"
|
|
261
|
-
end
|
|
262
164
|
# Continue in unmatched mode for remaining bits
|
|
263
165
|
break if symbol >= 0x100
|
|
264
166
|
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
if trace_233
|
|
268
|
-
puts "\n FINAL RESULT (after unmatched tail): 0x#{result.to_s(16).upcase} ('#{result.chr}')"
|
|
269
|
-
puts " Result bits: #{result_bits.join}"
|
|
270
|
-
puts "=== END MATCHED LITERAL TRACE ===\n"
|
|
271
|
-
end
|
|
272
|
-
return result
|
|
167
|
+
return decode_unmatched_tail(symbol, base_offset, lc,
|
|
168
|
+
range_decoder, models)
|
|
273
169
|
end
|
|
274
170
|
|
|
275
171
|
# Done when symbol reaches 0x100
|
|
276
172
|
break if symbol >= 0x100
|
|
277
|
-
|
|
278
|
-
bit_num += 1
|
|
279
173
|
end
|
|
280
174
|
|
|
281
|
-
|
|
282
|
-
if trace_233 || (ENV.fetch("TRACE_MATCHED_DECODE",
|
|
283
|
-
nil) && lit_state == 96)
|
|
284
|
-
puts "\n FINAL RESULT: 0x#{result.to_s(16).upcase} ('#{result.chr}')"
|
|
285
|
-
if trace_233
|
|
286
|
-
puts " Result bits: #{result_bits.join}"
|
|
287
|
-
end
|
|
288
|
-
puts "=== END MATCHED LITERAL DECODE ===\n"
|
|
289
|
-
end
|
|
290
|
-
result
|
|
175
|
+
symbol - 0x100
|
|
291
176
|
end
|
|
292
177
|
|
|
293
178
|
private
|