omnizip 0.3.8 → 0.3.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop_todo.yml +51 -50
- data/Gemfile +2 -0
- data/Rakefile +3 -0
- data/TODO.refactor/00-overview.md +67 -0
- data/TODO.refactor/01-registry-base-class.md +81 -0
- data/TODO.refactor/02-error-hierarchy.md +57 -0
- data/TODO.refactor/03-autoload-entry-point.md +69 -0
- data/TODO.refactor/04-filter-base-consolidation.md +49 -0
- data/TODO.refactor/05-send-private-methods.md +64 -0
- data/TODO.refactor/06-instance-variable-access.md +69 -0
- data/TODO.refactor/07-respond-to-replacement.md +91 -0
- data/TODO.refactor/08-spec-doubles.md +27 -0
- data/TODO.refactor/09-cli-shared-module.md +35 -0
- data/TODO.refactor/10-format-detector-ocp.md +34 -0
- data/TODO.refactor/11-convenience-decoupling.md +38 -0
- data/TODO.refactor/12-thread-safety.md +8 -0
- data/TODO.refactor/13-lutaml-model-migration.md +82 -0
- data/TODO.refactor/14-add-missing-specs.md +25 -0
- data/lib/omnizip/algorithm.rb +0 -2
- data/lib/omnizip/algorithm_registry.rb +29 -57
- data/lib/omnizip/algorithms/lzma/bit_model.rb +1 -1
- data/lib/omnizip/algorithms/lzma/dictionary.rb +4 -3
- data/lib/omnizip/algorithms/lzma/distance_coder.rb +5 -102
- data/lib/omnizip/algorithms/lzma/length_coder.rb +4 -92
- data/lib/omnizip/algorithms/lzma/literal_decoder.rb +4 -119
- data/lib/omnizip/algorithms/lzma/lzip_decoder.rb +2 -0
- data/lib/omnizip/algorithms/lzma/match_finder.rb +59 -29
- data/lib/omnizip/algorithms/lzma/optimal_encoder.rb +20 -12
- data/lib/omnizip/algorithms/lzma/range_decoder.rb +77 -269
- data/lib/omnizip/algorithms/lzma/range_encoder.rb +0 -9
- data/lib/omnizip/algorithms/lzma/xz_range_encoder_exact.rb +0 -15
- data/lib/omnizip/algorithms/lzma/xz_state.rb +0 -8
- data/lib/omnizip/algorithms/lzma/xz_utils_decoder.rb +189 -968
- data/lib/omnizip/algorithms/lzma2/encoder.rb +0 -2
- data/lib/omnizip/algorithms/lzma2/lzma2_chunk.rb +0 -2
- data/lib/omnizip/algorithms/lzma2/properties.rb +0 -2
- data/lib/omnizip/algorithms/ppmd7/model.rb +1 -1
- data/lib/omnizip/algorithms/ppmd8/context.rb +0 -2
- data/lib/omnizip/algorithms/ppmd8/model.rb +1 -1
- data/lib/omnizip/algorithms/ppmd8.rb +0 -2
- data/lib/omnizip/algorithms/ppmd_base.rb +0 -2
- data/lib/omnizip/algorithms/sevenzip_lzma2.rb +0 -5
- data/lib/omnizip/algorithms/xz_lzma2.rb +0 -5
- data/lib/omnizip/algorithms/zstandard/frame/header.rb +3 -5
- data/lib/omnizip/algorithms.rb +1 -0
- data/lib/omnizip/archive_handler.rb +75 -0
- data/lib/omnizip/archive_handlers/tar_handler.rb +51 -0
- data/lib/omnizip/archive_handlers/zip_handler.rb +73 -0
- data/lib/omnizip/archive_handlers.rb +11 -0
- data/lib/omnizip/checksum_registry.rb +23 -96
- data/lib/omnizip/cli/shared.rb +31 -0
- data/lib/omnizip/cli.rb +6 -43
- data/lib/omnizip/commands/archive_create_command.rb +0 -6
- data/lib/omnizip/commands/archive_extract_command.rb +0 -4
- data/lib/omnizip/commands/archive_list_command.rb +0 -4
- data/lib/omnizip/commands/archive_repair_command.rb +0 -2
- data/lib/omnizip/commands/archive_verify_command.rb +0 -2
- data/lib/omnizip/commands/compress_command.rb +0 -2
- data/lib/omnizip/commands/decompress_command.rb +0 -2
- data/lib/omnizip/commands/list_command.rb +0 -2
- data/lib/omnizip/commands/metadata_command.rb +0 -5
- data/lib/omnizip/commands/parity_create_command.rb +0 -2
- data/lib/omnizip/commands/parity_repair_command.rb +0 -2
- data/lib/omnizip/commands/parity_verify_command.rb +0 -2
- data/lib/omnizip/commands/profile_list_command.rb +0 -2
- data/lib/omnizip/commands/profile_show_command.rb +0 -2
- data/lib/omnizip/convenience.rb +111 -234
- data/lib/omnizip/converter/seven_zip_to_zip_strategy.rb +1 -1
- data/lib/omnizip/crypto/aes256/cipher.rb +6 -1
- data/lib/omnizip/error.rb +22 -18
- data/lib/omnizip/filter.rb +5 -3
- data/lib/omnizip/filter_registry.rb +74 -98
- data/lib/omnizip/filters/bcj2/decoder.rb +196 -130
- data/lib/omnizip/filters/bcj2.rb +0 -2
- data/lib/omnizip/filters/bcj_x86.rb +0 -2
- data/lib/omnizip/filters/filter_base.rb +4 -70
- data/lib/omnizip/filters/registry.rb +0 -2
- data/lib/omnizip/format_detector.rb +21 -12
- data/lib/omnizip/format_registry.rb +79 -38
- data/lib/omnizip/formats/bzip2_file.rb +0 -2
- data/lib/omnizip/formats/cpio/entry.rb +0 -2
- data/lib/omnizip/formats/cpio/reader.rb +0 -1
- data/lib/omnizip/formats/cpio/writer.rb +0 -1
- data/lib/omnizip/formats/cpio.rb +0 -1
- data/lib/omnizip/formats/format_spec_loader.rb +1 -1
- data/lib/omnizip/formats/gzip.rb +0 -2
- data/lib/omnizip/formats/iso/directory_record.rb +6 -3
- data/lib/omnizip/formats/iso/reader.rb +2 -2
- data/lib/omnizip/formats/iso.rb +0 -1
- data/lib/omnizip/formats/lzip.rb +5 -25
- data/lib/omnizip/formats/lzma_alone.rb +7 -25
- data/lib/omnizip/formats/msi/cab_extractor.rb +181 -0
- data/lib/omnizip/formats/msi/constants.rb +157 -0
- data/lib/omnizip/formats/msi/directory_resolver.rb +188 -0
- data/lib/omnizip/formats/msi/entry.rb +128 -0
- data/lib/omnizip/formats/msi/reader.rb +301 -0
- data/lib/omnizip/formats/msi/string_pool.rb +246 -0
- data/lib/omnizip/formats/msi/table_parser.rb +244 -0
- data/lib/omnizip/formats/msi.rb +84 -0
- data/lib/omnizip/formats/ole/allocation_table.rb +10 -1
- data/lib/omnizip/formats/ole/dirent.rb +20 -3
- data/lib/omnizip/formats/ole/header.rb +0 -1
- data/lib/omnizip/formats/ole/ranges_io.rb +21 -3
- data/lib/omnizip/formats/ole/storage.rb +19 -4
- data/lib/omnizip/formats/ole.rb +0 -2
- data/lib/omnizip/formats/rar/block_parser.rb +0 -1
- data/lib/omnizip/formats/rar/compression/ppmd/encoder.rb +1 -1
- data/lib/omnizip/formats/rar/decompressor.rb +38 -1
- data/lib/omnizip/formats/rar/header.rb +0 -1
- data/lib/omnizip/formats/rar/rar5/writer.rb +0 -2
- data/lib/omnizip/formats/rar/reader.rb +2 -3
- data/lib/omnizip/formats/rar/writer.rb +0 -1
- data/lib/omnizip/formats/rar.rb +2 -2
- data/lib/omnizip/formats/rpm/entry.rb +0 -1
- data/lib/omnizip/formats/rpm/header.rb +3 -8
- data/lib/omnizip/formats/rpm/lead.rb +37 -26
- data/lib/omnizip/formats/rpm/tag.rb +0 -1
- data/lib/omnizip/formats/rpm/writer.rb +0 -1
- data/lib/omnizip/formats/seven_zip/bcj2_stream_decompressor.rb +98 -61
- data/lib/omnizip/formats/seven_zip/coder_chain.rb +1 -2
- data/lib/omnizip/formats/seven_zip/encoded_header.rb +0 -1
- data/lib/omnizip/formats/seven_zip/encrypted_header.rb +0 -1
- data/lib/omnizip/formats/seven_zip/header.rb +0 -1
- data/lib/omnizip/formats/seven_zip/header_encryptor.rb +0 -1
- data/lib/omnizip/formats/seven_zip/header_writer.rb +22 -61
- data/lib/omnizip/formats/seven_zip/models/file_entry.rb +1 -1
- data/lib/omnizip/formats/seven_zip/parser.rb +14 -13
- data/lib/omnizip/formats/seven_zip/reader.rb +82 -19
- data/lib/omnizip/formats/seven_zip/split_archive_reader.rb +0 -15
- data/lib/omnizip/formats/seven_zip/split_archive_writer.rb +0 -1
- data/lib/omnizip/formats/seven_zip/writer.rb +22 -68
- data/lib/omnizip/formats/seven_zip.rb +63 -5
- data/lib/omnizip/formats/tar/entry.rb +0 -2
- data/lib/omnizip/formats/tar/header.rb +0 -2
- data/lib/omnizip/formats/tar/reader.rb +0 -2
- data/lib/omnizip/formats/tar/writer.rb +0 -2
- data/lib/omnizip/formats/tar.rb +0 -1
- data/lib/omnizip/formats/xar/entry.rb +0 -1
- data/lib/omnizip/formats/xar/header.rb +0 -1
- data/lib/omnizip/formats/xar/reader.rb +0 -1
- data/lib/omnizip/formats/xar/toc.rb +3 -3
- data/lib/omnizip/formats/xar/writer.rb +11 -3
- data/lib/omnizip/formats/xar.rb +0 -1
- data/lib/omnizip/formats/xz/reader.rb +7 -3
- data/lib/omnizip/formats/xz.rb +3 -40
- data/lib/omnizip/formats/xz_impl/block_decoder.rb +4 -10
- data/lib/omnizip/formats/xz_impl/block_encoder.rb +0 -2
- data/lib/omnizip/formats/xz_impl/block_header_parser.rb +0 -2
- data/lib/omnizip/formats/xz_impl/index_decoder.rb +0 -1
- data/lib/omnizip/formats/xz_impl/index_encoder.rb +0 -1
- data/lib/omnizip/formats/xz_impl/stream_encoder.rb +1 -1
- data/lib/omnizip/formats/xz_impl/stream_footer.rb +0 -1
- data/lib/omnizip/formats/xz_impl/stream_footer_parser.rb +0 -1
- data/lib/omnizip/formats/xz_impl/stream_header.rb +0 -1
- data/lib/omnizip/formats/xz_impl/stream_header_parser.rb +0 -2
- data/lib/omnizip/formats/zip/central_directory_header.rb +2 -3
- data/lib/omnizip/formats/zip/end_of_central_directory.rb +0 -2
- data/lib/omnizip/formats/zip/local_file_header.rb +0 -2
- data/lib/omnizip/formats/zip/reader.rb +12 -2
- data/lib/omnizip/formats/zip/writer.rb +73 -1
- data/lib/omnizip/formats/zip/zip64_end_of_central_directory.rb +0 -2
- data/lib/omnizip/formats/zip/zip64_end_of_central_directory_locator.rb +0 -2
- data/lib/omnizip/formats/zip/zip64_extra_field.rb +0 -2
- data/lib/omnizip/formats/zip.rb +0 -1
- data/lib/omnizip/formats.rb +9 -22
- data/lib/omnizip/implementations/seven_zip/lzma/range_decoder.rb +2 -2
- data/lib/omnizip/implementations/seven_zip/lzma/range_encoder.rb +1 -1
- data/lib/omnizip/implementations/seven_zip/lzma2/encoder.rb +0 -6
- data/lib/omnizip/implementations/xz_utils/lzma2/decoder.rb +18 -185
- data/lib/omnizip/implementations/xz_utils/lzma2/encoder.rb +3 -3
- data/lib/omnizip/io/source.rb +136 -0
- data/lib/omnizip/io.rb +2 -0
- data/lib/omnizip/link_handler.rb +7 -3
- data/lib/omnizip/metadata/metadata_editor.rb +1 -1
- data/lib/omnizip/models/algorithm_metadata.rb +33 -60
- data/lib/omnizip/models/compression_options.rb +40 -53
- data/lib/omnizip/models/conversion_options.rb +9 -18
- data/lib/omnizip/models/parallel_options.rb +25 -0
- data/lib/omnizip/optimization_registry.rb +8 -44
- data/lib/omnizip/parallel/job_scheduler.rb +26 -8
- data/lib/omnizip/parallel/parallel_compressor.rb +10 -20
- data/lib/omnizip/parallel/parallel_extractor.rb +20 -11
- data/lib/omnizip/parity/par2_repairer.rb +5 -5
- data/lib/omnizip/parity/par2_verifier.rb +27 -22
- data/lib/omnizip/parity.rb +4 -4
- data/lib/omnizip/password/encryption_registry.rb +11 -50
- data/lib/omnizip/password/zip_crypto_strategy.rb +1 -4
- data/lib/omnizip/pipe/stream_decompressor.rb +2 -3
- data/lib/omnizip/profile/archive_profile.rb +0 -2
- data/lib/omnizip/profile/binary_profile.rb +0 -2
- data/lib/omnizip/profile/compression_profile.rb +0 -1
- data/lib/omnizip/profile/profile_detector.rb +0 -2
- data/lib/omnizip/profile/text_profile.rb +0 -2
- data/lib/omnizip/profile.rb +0 -2
- data/lib/omnizip/registry.rb +112 -0
- data/lib/omnizip/version.rb +1 -1
- data/lib/omnizip/zip/file.rb +20 -9
- data/lib/omnizip.rb +26 -55
- data/lib/tasks/msi_test_files.rake +66 -0
- metadata +57 -6
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "stringio"
|
|
4
|
+
|
|
3
5
|
# Copyright (C) 2025 Ribose Inc.
|
|
4
6
|
#
|
|
5
7
|
# Permission is hereby granted, free of charge, to any person obtaining a
|
|
@@ -40,13 +42,6 @@ module Omnizip
|
|
|
40
42
|
# - XZ format (.xz files)
|
|
41
43
|
# - LZMA2 compression in XZ format
|
|
42
44
|
#
|
|
43
|
-
module XzUtilsDecoderDebug
|
|
44
|
-
# Debug helper to conditionally output debug messages
|
|
45
|
-
# Set ENV['LZMA_DEBUG'] = 'true' to enable all debug output
|
|
46
|
-
def self.debug_puts(*args)
|
|
47
|
-
puts(*args) if ENV["LZMA_DEBUG"]
|
|
48
|
-
end
|
|
49
|
-
end
|
|
50
45
|
|
|
51
46
|
# XZ Utils implementation of LZMA decoder
|
|
52
47
|
#
|
|
@@ -91,13 +86,58 @@ module Omnizip
|
|
|
91
86
|
RangeDecoder = LZMA::RangeDecoder
|
|
92
87
|
SdkStateMachine = Implementations::SevenZip::LZMA::StateMachine
|
|
93
88
|
|
|
94
|
-
attr_reader :lc, :lp, :pb, :dict_size, :uncompressed_size
|
|
89
|
+
attr_reader :lc, :lp, :pb, :dict_size, :uncompressed_size,
|
|
90
|
+
:range_decoder
|
|
95
91
|
|
|
96
92
|
# XZ Utils dictionary constants (from lz_decoder.h)
|
|
97
93
|
# See: /Users/mulgogi/src/external/xz/src/liblzma/lz/lz_decoder.h
|
|
98
94
|
LZ_DICT_REPEAT_MAX = 288
|
|
99
95
|
LZ_DICT_INIT_POS = 2 * LZ_DICT_REPEAT_MAX # = 576
|
|
100
96
|
|
|
97
|
+
# Map a monotonic position to a circular buffer index.
|
|
98
|
+
# Buffer layout: [LZ_DICT_INIT_POS zero bytes] + [dict_size circular region]
|
|
99
|
+
# @pos grows forever; this maps it to the fixed-size buffer.
|
|
100
|
+
# Matches C++ 7-Zip SDK: dic[dicPos] with dicPos wrapping at dicBufSize.
|
|
101
|
+
def dict_index(pos)
|
|
102
|
+
LZ_DICT_INIT_POS + ((pos - LZ_DICT_INIT_POS) % @dict_size)
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# Flush decoded output from the circular dictionary buffer.
|
|
106
|
+
# Copies bytes from @flush_pos to @pos into the output stream or accumulator.
|
|
107
|
+
# Handles wrap-around when the data spans the circular buffer boundary.
|
|
108
|
+
def flush_circular_output(output)
|
|
109
|
+
return if @pos == @flush_pos
|
|
110
|
+
|
|
111
|
+
length = @pos - @flush_pos
|
|
112
|
+
|
|
113
|
+
if length > @dict_size
|
|
114
|
+
raise Omnizip::DecompressionError,
|
|
115
|
+
"Internal error: flush lag (#{length}) exceeds dict_size (#{@dict_size})"
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
start_idx = dict_index(@flush_pos)
|
|
119
|
+
buf_end = @buf_end
|
|
120
|
+
|
|
121
|
+
if start_idx + length <= buf_end
|
|
122
|
+
# No wrap
|
|
123
|
+
data = @dict_buf.byteslice(start_idx, length)
|
|
124
|
+
else
|
|
125
|
+
# Wraps around the circular boundary
|
|
126
|
+
first_len = buf_end - start_idx
|
|
127
|
+
data = @dict_buf.byteslice(start_idx, first_len)
|
|
128
|
+
data << @dict_buf.byteslice(LZ_DICT_INIT_POS, length - first_len)
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
if output
|
|
132
|
+
output.write(data.force_encoding(Encoding::BINARY))
|
|
133
|
+
else
|
|
134
|
+
@output_accumulator ||= StringIO.new("".b)
|
|
135
|
+
@output_accumulator.write(data)
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
@flush_pos = @pos
|
|
139
|
+
end
|
|
140
|
+
|
|
101
141
|
# Initialize the SDK-compatible decoder
|
|
102
142
|
#
|
|
103
143
|
# @param input [IO] Input stream of compressed data
|
|
@@ -116,11 +156,6 @@ module Omnizip
|
|
|
116
156
|
def initialize(input, options = {})
|
|
117
157
|
@input = input
|
|
118
158
|
@decoder_id = object_id # Track decoder instance ID
|
|
119
|
-
puts "DEBUG LZMA::Decoder.new created[#{@decoder_id}]" if ENV["LZMA_DEBUG_DECODE_STREAM"]
|
|
120
|
-
if ENV["LZMA_DEBUG_DECODE_STREAM"]
|
|
121
|
-
warn "SDK Decoder #{@decoder_id} created"
|
|
122
|
-
end
|
|
123
|
-
@input = input
|
|
124
159
|
|
|
125
160
|
# Check for preloaded data (from LZMA2 uncompressed chunks)
|
|
126
161
|
@preloaded_data = options[:preloaded_data]
|
|
@@ -140,9 +175,15 @@ module Omnizip
|
|
|
140
175
|
# Standalone LZMA file - read header from input
|
|
141
176
|
read_header
|
|
142
177
|
end
|
|
178
|
+
|
|
143
179
|
validate_parameters
|
|
144
180
|
init_models
|
|
145
181
|
init_coders
|
|
182
|
+
|
|
183
|
+
# Cache computed values for hot loop
|
|
184
|
+
@pb_mask = (1 << @pb) - 1
|
|
185
|
+
@pb_shift = 1 << @pb
|
|
186
|
+
@literal_mask = (0x100 << @lp) - (0x100 >> @lc)
|
|
146
187
|
end
|
|
147
188
|
|
|
148
189
|
# Decode a compressed stream
|
|
@@ -167,46 +208,17 @@ module Omnizip
|
|
|
167
208
|
check_rc_finished: true)
|
|
168
209
|
@decode_stream_call_count ||= 0
|
|
169
210
|
@decode_stream_call_count += 1
|
|
170
|
-
call_num = @decode_stream_call_count
|
|
171
|
-
|
|
172
|
-
puts "DEBUG decode_stream START (call ##{call_num}): @pos=#{@pos.inspect}, @dict_full=#{@dict_full.inspect}, preserve_dict=#{preserve_dict}, @uncompressed_size=#{@uncompressed_size.inspect}" if ENV.fetch(
|
|
173
|
-
"LZMA_DEBUG", nil
|
|
174
|
-
) && @dict_full && @dict_full >= 200 && @dict_full <= 230
|
|
175
|
-
if ENV["LZMA_DEBUG_DECODE_STREAM"]
|
|
176
|
-
warn "DEBUG decode_stream[#{@decoder_id}] START: preserve_dict=#{preserve_dict}, @pos=#{@pos.inspect}, @dict_full=#{@dict_full.inspect}, @dict_buf.object_id=#{@dict_buf&.object_id || 'nil'}, @dict_buf.size=#{@dict_buf&.size || 'nil'}"
|
|
177
|
-
end
|
|
178
|
-
|
|
179
|
-
# Initialize range decoder
|
|
180
|
-
# For LZMA2, reuse persistent range decoder across chunks (like XZ Utils)
|
|
181
|
-
# The range decoder is created in set_input when the first chunk is processed
|
|
182
|
-
# See: /Users/mulgogi/src/external/xz/src/liblzma/lzma/lzma2_decoder.c:140-141
|
|
183
|
-
if ENV["LZMA_DEBUG"]
|
|
184
|
-
warn "DEBUG: decode_stream - reusing range decoder @input.pos=#{begin
|
|
185
|
-
@input.pos
|
|
186
|
-
rescue StandardError
|
|
187
|
-
'N/A'
|
|
188
|
-
end}, @range_decoder.class=#{@range_decoder.class}"
|
|
189
|
-
end
|
|
190
211
|
|
|
191
212
|
# Create range decoder if it doesn't exist (first chunk)
|
|
192
213
|
# This happens when the decoder is created directly for LZMA (not LZMA2)
|
|
193
|
-
|
|
194
|
-
if ENV["LZMA_DEBUG"]
|
|
195
|
-
warn "DEBUG: decode_stream - creating NEW range decoder"
|
|
196
|
-
end
|
|
197
|
-
@range_decoder = RangeDecoder.new(@input)
|
|
198
|
-
end
|
|
214
|
+
@range_decoder ||= RangeDecoder.new(@input)
|
|
199
215
|
|
|
200
216
|
# Special case: empty input (uncompressed_size == 0)
|
|
201
217
|
# Return immediately without trying to decode anything
|
|
202
218
|
if @uncompressed_size != 0xFFFFFFFFFFFFFFFF && @uncompressed_size.zero?
|
|
203
|
-
if ENV["LZMA_DEBUG"]
|
|
204
|
-
warn "DEBUG: decode_stream - empty input (uncompressed_size=0), returning immediately"
|
|
205
|
-
end
|
|
206
219
|
return "" # Empty output
|
|
207
220
|
end
|
|
208
221
|
|
|
209
|
-
# TEMP DEBUG: Trace first 10 iterations
|
|
210
222
|
@debug_iter = 0
|
|
211
223
|
|
|
212
224
|
# Track bytes decoded in this chunk (for multi-chunk streams)
|
|
@@ -214,13 +226,6 @@ check_rc_finished: true)
|
|
|
214
226
|
# represents only the current chunk's size, not the total size
|
|
215
227
|
@chunk_bytes_decoded = 0
|
|
216
228
|
|
|
217
|
-
# DEBUG: Show chunk_bytes_decoded initialization
|
|
218
|
-
if @dict_full && @dict_full >= 220 && @dict_full <= 240 && ENV.fetch(
|
|
219
|
-
"LZMA_DEBUG", nil
|
|
220
|
-
)
|
|
221
|
-
puts "DEBUG: chunk_bytes_decoded reset to 0 for chunk (call_num=#{call_num}, dict_full=#{@dict_full})"
|
|
222
|
-
end
|
|
223
|
-
|
|
224
229
|
# Initialize state and dictionary (XZ Utils system from lz_decoder.c)
|
|
225
230
|
# See: /Users/mulgogi/src/external/xz/src/liblzma/lz/lz_decoder.c:56
|
|
226
231
|
# For LZMA2 multi-chunk streams, state machine persists across chunks
|
|
@@ -235,38 +240,31 @@ check_rc_finished: true)
|
|
|
235
240
|
# when preserve_dict is true (control >= 0x80 but < 0xA0)
|
|
236
241
|
# For subsequent chunks, the reset() method handles dictionary reset
|
|
237
242
|
# For the first chunk (when @dict_buf is nil), we need to init it here
|
|
238
|
-
puts "DEBUG: Checking @dict_buf.nil? = #{@dict_buf.nil?}, preserve_dict=#{preserve_dict}" if ENV["LZMA_DEBUG_RESET"]
|
|
239
243
|
if @dict_buf.nil?
|
|
240
|
-
|
|
241
|
-
|
|
244
|
+
@buf_end = LZ_DICT_INIT_POS + @dict_size
|
|
245
|
+
buf_size = @buf_end
|
|
246
|
+
@dict_buf = ("\0" * buf_size).b
|
|
242
247
|
@pos = LZ_DICT_INIT_POS
|
|
248
|
+
@dict_pos = LZ_DICT_INIT_POS # Circular buffer write position (tracks @pos)
|
|
243
249
|
@dict_full = 0
|
|
244
250
|
@has_wrapped = false
|
|
245
251
|
|
|
246
252
|
# Add preloaded data to dictionary (from LZMA2 uncompressed chunks)
|
|
247
253
|
# This must be done before decoding so matches can reference this data
|
|
248
254
|
if @preloaded_data && !@preloaded_data.empty?
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
end
|
|
255
|
+
idx = @dict_pos
|
|
256
|
+
buf_end = @buf_end
|
|
252
257
|
@preloaded_data.each_byte do |byte|
|
|
253
|
-
@dict_buf
|
|
258
|
+
@dict_buf.setbyte(idx, byte)
|
|
254
259
|
@pos += 1
|
|
260
|
+
idx += 1
|
|
261
|
+
idx = LZ_DICT_INIT_POS if idx >= buf_end
|
|
255
262
|
end
|
|
263
|
+
@dict_pos = idx
|
|
256
264
|
# Update dict_full to reflect preloaded data
|
|
257
265
|
@dict_full = @pos - LZ_DICT_INIT_POS
|
|
258
|
-
if ENV["LZMA_DEBUG_RESET"]
|
|
259
|
-
warn "DEBUG: After preload - @pos=#{@pos}, @dict_full=#{@dict_full}"
|
|
260
|
-
warn " Preloaded data (hex): #{@preloaded_data[0..50].unpack1('H*')}"
|
|
261
|
-
end
|
|
262
266
|
@preloaded_data = nil # Clear after loading
|
|
263
267
|
end
|
|
264
|
-
|
|
265
|
-
if ENV["LZMA_DEBUG_RESET"]
|
|
266
|
-
warn "DEBUG: Dictionary init in decode_stream[#{@decoder_id}] - @pos=#{@pos}, @dict_full=#{@dict_full}, @dict_buf.size=#{@dict_buf.size}, @dict_buf.object_id=#{@dict_buf.object_id}"
|
|
267
|
-
# Verify buffer initialization by checking a few positions
|
|
268
|
-
warn " Sample values: [576]=#{@dict_buf[576]}, [577]=#{@dict_buf[577]}, [578]=#{@dict_buf[578]}, [583]=#{@dict_buf[583]}"
|
|
269
|
-
end
|
|
270
268
|
end
|
|
271
269
|
|
|
272
270
|
# Track starting position for multi-chunk streams
|
|
@@ -275,13 +273,6 @@ check_rc_finished: true)
|
|
|
275
273
|
# properly reflected in start_pos, so we only return NEW bytes.
|
|
276
274
|
# For LZMA2, we need to return only the NEW bytes, not all bytes from LZ_DICT_INIT_POS
|
|
277
275
|
start_pos = @pos || LZ_DICT_INIT_POS
|
|
278
|
-
puts "DEBUG: start_pos=#{start_pos}, @pos=#{@pos.inspect}, @dict_full=#{@dict_full.inspect}, preserve_dict=#{preserve_dict}, @decoder_id=#{@decoder_id}" if ENV.fetch(
|
|
279
|
-
"LZMA_DEBUG", nil
|
|
280
|
-
) && @dict_full && @dict_full >= 200 && @dict_full <= 230
|
|
281
|
-
# Also show for chunk #1 start (dict_full around 227)
|
|
282
|
-
puts "DEBUG: start_pos=#{start_pos}, @pos=#{@pos.inspect}, @dict_full=#{@dict_full.inspect}, @uncompressed_size=#{@uncompressed_size}, @decoder_id=#{@decoder_id}" if ENV.fetch(
|
|
283
|
-
"LZMA_DEBUG", nil
|
|
284
|
-
) && @dict_full && @dict_full >= 225 && @dict_full <= 230
|
|
285
276
|
|
|
286
277
|
# Initialize rep distances (XZ Utils initializes to 0)
|
|
287
278
|
# See: /Users/mulgogi/src/external/xz/src/liblzma/lzma/lzma_decoder.c:1054-1055
|
|
@@ -290,15 +281,17 @@ check_rc_finished: true)
|
|
|
290
281
|
#
|
|
291
282
|
# IMPORTANT: Initialize rep distances if they're nil OR not preserving dict
|
|
292
283
|
if @rep0.nil? || @rep1.nil? || @rep2.nil? || @rep3.nil? || !preserve_dict
|
|
293
|
-
puts "DEBUG: Resetting rep distances to 0 (rep0.nil?=#{@rep0.nil?}, preserve_dict=#{preserve_dict})" if ENV.fetch(
|
|
294
|
-
"LZMA_DEBUG", nil
|
|
295
|
-
) && @dict_full && @dict_full >= 200 && @dict_full <= 230
|
|
296
284
|
@rep0 = 0
|
|
297
285
|
@rep1 = 0
|
|
298
286
|
@rep2 = 0
|
|
299
287
|
@rep3 = 0
|
|
300
288
|
end
|
|
301
289
|
|
|
290
|
+
# Read range decoder init bytes (must happen after set_input sets correct stream)
|
|
291
|
+
if @range_decoder.respond_to?(:read_init_bytes) && @range_decoder.init_bytes_remaining.positive?
|
|
292
|
+
@range_decoder.read_init_bytes
|
|
293
|
+
end
|
|
294
|
+
|
|
302
295
|
# Main decoding loop
|
|
303
296
|
# XZ Utils pattern (lzma_decoder.c:305-306):
|
|
304
297
|
# Set dict.limit = dict.pos + (size_t)(coder->uncompressed_size)
|
|
@@ -313,178 +306,42 @@ check_rc_finished: true)
|
|
|
313
306
|
start_pos + @uncompressed_size
|
|
314
307
|
end
|
|
315
308
|
|
|
316
|
-
#
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
# DEBUG: Also show for dict_full around 293 (where the error occurs)
|
|
322
|
-
if ENV.fetch("LZMA_DEBUG_LIMIT",
|
|
323
|
-
nil) && @dict_full && @dict_full >= 290 && @dict_full <= 300
|
|
324
|
-
puts "DEBUG LIMIT CALCULATION at dict_full=#{@dict_full}: start_pos=#{start_pos}, @uncompressed_size=#{@uncompressed_size}, limit=#{limit.inspect}, @decoder_id=#{@decoder_id}"
|
|
325
|
-
end
|
|
309
|
+
# Circular buffer: no growth needed. Buffer is fixed at dict_size + LZ_DICT_INIT_POS.
|
|
310
|
+
# Output is flushed incrementally when the circular position nears wrap point.
|
|
311
|
+
# Matches C++ 7-Zip SDK: dic is pre-allocated to dicBufSize and never resized.
|
|
312
|
+
@flush_pos = @pos
|
|
313
|
+
@output_accumulator = output.nil? ? StringIO.new("".b) : nil
|
|
326
314
|
|
|
327
315
|
iteration = 0
|
|
328
316
|
loop do
|
|
329
317
|
iteration += 1
|
|
330
|
-
# DEBUG: Show every iteration after position 200
|
|
331
|
-
if ENV.fetch("LZMA_DEBUG_ITER",
|
|
332
|
-
nil) && @dict_full && @dict_full >= 200 && @dict_full <= 500
|
|
333
|
-
puts "DEBUG ITERATION ##{iteration}: pos=#{@pos}, dict_full=#{@dict_full}, limit=#{limit.inspect}"
|
|
334
|
-
end
|
|
335
318
|
# Check if we've reached the expected size (if known)
|
|
336
319
|
# XZ Utils: checks dict.pos < dict.limit
|
|
337
|
-
if ENV["LZMA_DEBUG_LIMIT"]
|
|
338
|
-
compare_result = begin
|
|
339
|
-
limit && @pos >= limit
|
|
340
|
-
rescue StandardError
|
|
341
|
-
"ERROR"
|
|
342
|
-
end
|
|
343
|
-
XzUtilsDecoderDebug.debug_puts "DEBUG LIMIT: iter=#{iteration}, pos=#{@pos.inspect}, dict_full=#{@dict_full}, limit=#{limit.inspect}, pos >= limit: #{compare_result}"
|
|
344
|
-
end
|
|
345
320
|
|
|
346
321
|
# Handle nil @pos or limit gracefully
|
|
347
322
|
if limit && (@pos.nil? || limit.nil?)
|
|
348
323
|
raise "Invalid state: @pos=#{@pos.inspect}, limit=#{limit.inspect}"
|
|
349
324
|
end
|
|
350
325
|
|
|
351
|
-
|
|
352
|
-
|
|
326
|
+
# Circular buffer: flush output before it can be overwritten.
|
|
327
|
+
# LZMA matches are at most 273 bytes, so flush when within 273 of wrapping.
|
|
328
|
+
if @pos - @flush_pos > @dict_size - 273
|
|
329
|
+
flush_circular_output(output)
|
|
353
330
|
end
|
|
354
331
|
|
|
355
|
-
# DEBUG: Track position before decoding
|
|
356
|
-
@pos if ENV["LZMA_DEBUG_POS"]
|
|
357
|
-
|
|
358
332
|
# Decode is_match bit
|
|
359
|
-
pos_state = @pos &
|
|
333
|
+
pos_state = @pos & @pb_mask
|
|
360
334
|
# XZ Utils: is_match[state][pos_state] where the array is NUM_STATES * (1 << pb)
|
|
361
335
|
# The array stride changes with pb value
|
|
362
|
-
model_index = (@state.value *
|
|
336
|
+
model_index = (@state.value * @pb_shift) + pos_state
|
|
363
337
|
|
|
364
|
-
# DEBUG: Show state before decode (for position tracking)
|
|
365
|
-
if ENV["LZMA_DEBUG_POS_227"]
|
|
366
|
-
XzUtilsDecoderDebug.debug_puts "DEBUG: Before is_match at pos=#{@pos}, dict_full=#{@dict_full}, state=#{@state.value}, pos_state=#{pos_state}"
|
|
367
|
-
end
|
|
368
|
-
|
|
369
|
-
# Debug trace (disabled - remove or enable with ENV var as needed)
|
|
370
338
|
@debug_iter += 1
|
|
371
339
|
|
|
372
|
-
# DEBUG: Trace is_match decision around position 256
|
|
373
|
-
# IMPORTANT: Capture range/code BEFORE calling decode_bit
|
|
374
|
-
if @dict_full.between?(255, 257)
|
|
375
|
-
range = @range_decoder.instance_variable_get(:@range)
|
|
376
|
-
code = @range_decoder.instance_variable_get(:@code)
|
|
377
|
-
model = @is_match_models[model_index]
|
|
378
|
-
XzUtilsDecoderDebug.debug_puts " [IS_MATCH] pos=#{@pos}, dict_full=#{@dict_full}, state=#{@state.value}, pos_state=#{pos_state}, model_index=#{model_index}"
|
|
379
|
-
XzUtilsDecoderDebug.debug_puts " BEFORE decode: range=0x#{range.to_s(16)}, code=0x#{code.to_s(16)}, prob=#{model.probability}"
|
|
380
|
-
# Enable detailed tracing for the critical is_match[0][0] decision
|
|
381
|
-
$trace_is_match_0_0 = (@dict_full == 256)
|
|
382
|
-
end
|
|
383
|
-
|
|
384
|
-
if ENV.fetch("LZMA_DEBUG_ITER", nil)
|
|
385
|
-
range = @range_decoder.instance_variable_get(:@range)
|
|
386
|
-
code = @range_decoder.instance_variable_get(:@code)
|
|
387
|
-
model = @is_match_models[model_index]
|
|
388
|
-
bound = (range >> 11) * model.probability
|
|
389
|
-
XzUtilsDecoderDebug.debug_puts ""
|
|
390
|
-
XzUtilsDecoderDebug.debug_puts "ITER #{@debug_iter}:"
|
|
391
|
-
XzUtilsDecoderDebug.debug_puts " pos=#{@pos}, state=#{@state.value}, pos_state=#{pos_state}, model_index=#{model_index}"
|
|
392
|
-
XzUtilsDecoderDebug.debug_puts " dict_full=#{@dict_full}"
|
|
393
|
-
XzUtilsDecoderDebug.debug_puts " range=0x#{range.to_s(16)}, code=0x#{code.to_s(16)}, model.prob=#{model.probability}"
|
|
394
|
-
XzUtilsDecoderDebug.debug_puts " bound=0x#{bound.to_s(16)}, code < bound: #{code < bound}"
|
|
395
|
-
end
|
|
396
|
-
|
|
397
340
|
is_match = @range_decoder.decode_bit(@is_match_models[model_index])
|
|
398
341
|
|
|
399
|
-
# DEBUG: Trace is_match and literal/match decisions around dict_full = 50-62
|
|
400
|
-
if @dict_full.between?(50, 62)
|
|
401
|
-
range_val = @range_decoder.instance_variable_get(:@range)
|
|
402
|
-
code_val = @range_decoder.instance_variable_get(:@code)
|
|
403
|
-
prob_val = @is_match_models[model_index].probability
|
|
404
|
-
XzUtilsDecoderDebug.debug_puts "\n=== dict_full=#{@dict_full}: is_match=#{is_match}, state=#{@state.value}, pos_state=#{pos_state} ==="
|
|
405
|
-
XzUtilsDecoderDebug.debug_puts " model_index=#{model_index}, prob=#{prob_val}"
|
|
406
|
-
XzUtilsDecoderDebug.debug_puts " range=0x#{range_val.to_s(16).upcase}, code=0x#{code_val.to_s(16).upcase}"
|
|
407
|
-
end
|
|
408
|
-
|
|
409
|
-
if ENV.fetch("LZMA_DEBUG_ITER", nil)
|
|
410
|
-
XzUtilsDecoderDebug.debug_puts " is_match=#{is_match}"
|
|
411
|
-
end
|
|
412
|
-
|
|
413
|
-
# DEBUG: Show is_match result after decode
|
|
414
|
-
if @dict_full.between?(255, 257)
|
|
415
|
-
XzUtilsDecoderDebug.debug_puts " AFTER decode: is_match=#{is_match}"
|
|
416
|
-
XzUtilsDecoderDebug.debug_puts " (is_match=0 means literal, is_match=1 means match)"
|
|
417
|
-
end
|
|
418
|
-
|
|
419
|
-
# DEBUG: Track what's happening around dict_full=227 (corruption point)
|
|
420
|
-
if ENV["LZMA_DEBUG"] && @dict_full == 227
|
|
421
|
-
puts "DEBUG CORRUPTION POINT: dict_full=#{@dict_full}, pos=#{@pos}"
|
|
422
|
-
puts " is_match=#{is_match}, state=#{@state.value}"
|
|
423
|
-
range_val = @range_decoder.instance_variable_get(:@range)
|
|
424
|
-
code_val = @range_decoder.instance_variable_get(:@code)
|
|
425
|
-
puts " range=0x#{range_val.to_s(16)}, code=0x#{code_val.to_s(16)}"
|
|
426
|
-
puts " dict_buf[#{@pos - 5}...#{@pos + 5}] = #{@dict_buf[[
|
|
427
|
-
@pos - 5, LZ_DICT_INIT_POS
|
|
428
|
-
].max...[@pos + 5, @dict_buf.size - 1].min].inspect}"
|
|
429
|
-
end
|
|
430
|
-
|
|
431
|
-
if ENV["LZMA_DEBUG"] && @dict_full.between?(224, 235)
|
|
432
|
-
puts "DEBUG pos #{@dict_full}: is_match=#{is_match}, state=#{@state.value}"
|
|
433
|
-
if is_match.zero?
|
|
434
|
-
puts " Next byte should be literal"
|
|
435
|
-
else
|
|
436
|
-
puts " Next byte should be match"
|
|
437
|
-
end
|
|
438
|
-
end
|
|
439
|
-
|
|
440
|
-
# DEBUG: Verify first 256 bytes are correct
|
|
441
|
-
if @dict_full == 256
|
|
442
|
-
XzUtilsDecoderDebug.debug_puts ""
|
|
443
|
-
XzUtilsDecoderDebug.debug_puts " Verifying first 256 bytes:"
|
|
444
|
-
# Check specific bytes around position 253
|
|
445
|
-
XzUtilsDecoderDebug.debug_puts " Byte 253: @dict_buf[#{LZ_DICT_INIT_POS + 253}]=#{@dict_buf[LZ_DICT_INIT_POS + 253].inspect} (expected 'i'=0x69)"
|
|
446
|
-
XzUtilsDecoderDebug.debug_puts " Byte 254: @dict_buf[#{LZ_DICT_INIT_POS + 254}]=#{@dict_buf[LZ_DICT_INIT_POS + 254].inspect} (expected 'n'=0x6E)"
|
|
447
|
-
XzUtilsDecoderDebug.debug_puts " Byte 255: @dict_buf[#{LZ_DICT_INIT_POS + 255}]=#{@dict_buf[LZ_DICT_INIT_POS + 255].inspect} (expected ' '=0x20)"
|
|
448
|
-
all_correct = true
|
|
449
|
-
256.times do |i|
|
|
450
|
-
expected = i
|
|
451
|
-
actual = @dict_buf[LZ_DICT_INIT_POS + i]
|
|
452
|
-
if actual != expected
|
|
453
|
-
all_correct = false
|
|
454
|
-
if (i >= 253) && ENV.fetch("LZMA_DEBUG", nil)
|
|
455
|
-
puts " Byte #{i}: expected 0x#{expected.to_s(16)}, got 0x#{actual.to_s(16)} MISMATCH!"
|
|
456
|
-
end
|
|
457
|
-
end
|
|
458
|
-
end
|
|
459
|
-
XzUtilsDecoderDebug.debug_puts " First 256 bytes: #{all_correct ? 'ALL CORRECT ✓' : 'HAS MISMATCH'}"
|
|
460
|
-
XzUtilsDecoderDebug.debug_puts ""
|
|
461
|
-
end
|
|
462
|
-
|
|
463
|
-
if ENV.fetch("LZMA_DEBUG", nil) && @pos >= 605 && @pos <= 615
|
|
464
|
-
warn "DEBUG: is_match at pos=#{@pos}, state=#{@state.value}, pos_state=#{pos_state}, model_index=#{model_index}, is_match=#{is_match}"
|
|
465
|
-
end
|
|
466
|
-
|
|
467
342
|
if is_match.zero?
|
|
468
343
|
# Decode literal
|
|
469
344
|
decode_literal
|
|
470
|
-
|
|
471
|
-
# Trace positions 45-65 for debugging good-1-lzma2-3.xz divergence
|
|
472
|
-
if ENV.fetch("LZMA_DEBUG",
|
|
473
|
-
nil) && @dict_full >= 45 && @dict_full <= 65
|
|
474
|
-
last_byte = @dict_buf[@pos - 1]
|
|
475
|
-
range_after = @range_decoder.instance_variable_get(:@range)
|
|
476
|
-
code_after = @range_decoder.instance_variable_get(:@code)
|
|
477
|
-
puts " literal decoded: 0x#{last_byte.to_s(16).upcase} ('#{last_byte.chr}') at pos=#{@pos - 1}, dict_full=#{@dict_full}"
|
|
478
|
-
puts " AFTER: range=0x#{range_after.to_s(16).upcase}, code=0x#{code_after.to_s(16).upcase}"
|
|
479
|
-
end
|
|
480
|
-
|
|
481
|
-
if ENV.fetch("LZMA_DEBUG_ITER", nil)
|
|
482
|
-
last_byte = @dict_buf[@pos - 1]
|
|
483
|
-
puts " literal byte=0x#{last_byte.to_s(16)} ('#{last_byte.chr}')"
|
|
484
|
-
end
|
|
485
|
-
if ENV["LZMA_DEBUG_POS"] && @pos >= limit
|
|
486
|
-
puts "DEBUG: Literal overshoot: pos=#{@pos}, limit=#{limit}, delta=#{@pos - limit}"
|
|
487
|
-
end
|
|
488
345
|
elsif decode_match
|
|
489
346
|
# Decode match - returns true if EOS detected
|
|
490
347
|
break
|
|
@@ -496,7 +353,6 @@ check_rc_finished: true)
|
|
|
496
353
|
# IMPORTANT: Must verify range decoder is finished (code == 0)
|
|
497
354
|
# If code != 0, there's leftover data in the compressed stream (corruption)
|
|
498
355
|
if limit && @pos >= limit
|
|
499
|
-
puts "DEBUG LIMIT TRIGGERED (call #{call_num}): pos=#{@pos}, limit=#{limit}, dict_full=#{@dict_full}, chunk_bytes_decoded=#{@chunk_bytes_decoded}" if ENV["LZMA_DEBUG_LIMIT"]
|
|
500
356
|
|
|
501
357
|
# XZ Utils pattern (lzma_decoder.c:689-700):
|
|
502
358
|
# Check if range decoder is finished (code == 0)
|
|
@@ -531,32 +387,23 @@ check_rc_finished: true)
|
|
|
531
387
|
raise Omnizip::DecompressionError,
|
|
532
388
|
"LZMA stream finished with leftover compressed data (range_decoder.code=#{@range_decoder.code}, expected 0). This indicates corruption in the compressed stream or an invalid EOPM for LZMA2."
|
|
533
389
|
end
|
|
390
|
+
# XZ Utils pattern (lzma_decoder.c): when STREAM_END is reached,
|
|
391
|
+
# reset the range coder for the next chunk. This happens even for
|
|
392
|
+
# no-reset chunks (control 0x80-0x9F) - the range coder is ALWAYS
|
|
393
|
+
# re-initialized between chunks, only state/dict/models are preserved.
|
|
394
|
+
@range_decoder.reset
|
|
534
395
|
break
|
|
535
396
|
elsif @range_decoder.code.zero?
|
|
536
397
|
# EOPM is allowed (e.g., LZMA_Alone format)
|
|
537
398
|
# If range decoder is finished, we're done
|
|
399
|
+
# XZ Utils: rc_reset at STREAM_END
|
|
400
|
+
@range_decoder.reset
|
|
538
401
|
break
|
|
539
402
|
# Otherwise, continue decoding to find EOPM marker
|
|
540
403
|
# XZ Utils sets eopm_is_valid = true and continues
|
|
541
404
|
# Reference: lzma_decoder.c:704
|
|
542
405
|
end
|
|
543
406
|
end
|
|
544
|
-
|
|
545
|
-
# DEBUG: Show when approaching limit for chunk #1
|
|
546
|
-
if ENV.fetch("LZMA_DEBUG_LIMIT",
|
|
547
|
-
nil) && limit && @pos >= limit - 10 && @pos < limit + 10
|
|
548
|
-
puts "DEBUG NEAR LIMIT (call #{call_num}): pos=#{@pos}, limit=#{limit}, dict_full=#{@dict_full}, chunk_bytes_decoded=#{@chunk_bytes_decoded}, remaining=#{@uncompressed_size ? @uncompressed_size - @chunk_bytes_decoded : 'N/A'}"
|
|
549
|
-
end
|
|
550
|
-
|
|
551
|
-
# DEBUG: Show when we've passed the expected limit
|
|
552
|
-
if ENV.fetch("LZMA_DEBUG_LIMIT",
|
|
553
|
-
nil) && limit && @pos >= limit && @pos < limit + 10
|
|
554
|
-
puts "DEBUG PASSED LIMIT: pos=#{@pos}, limit=#{limit}, dict_full=#{@dict_full}, delta=#{@pos - limit}"
|
|
555
|
-
end
|
|
556
|
-
|
|
557
|
-
if ENV["LZMA_DEBUG_POS"] && @pos >= limit
|
|
558
|
-
XzUtilsDecoderDebug.debug_puts "DEBUG: Overshoot detected: pos=#{@pos}, limit=#{limit}, delta=#{@pos - limit}"
|
|
559
|
-
end
|
|
560
407
|
end
|
|
561
408
|
|
|
562
409
|
# Validate decoded size against expected uncompressed_size
|
|
@@ -585,8 +432,8 @@ check_rc_finished: true)
|
|
|
585
432
|
#
|
|
586
433
|
# If EOPM was NOT encountered (code != 0), leftover data is expected
|
|
587
434
|
# (it's part of the compressed stream that we haven't read yet).
|
|
588
|
-
if @allow_eopm && @range_decoder&.code&.zero? && @range_decoder
|
|
589
|
-
stream = @range_decoder.
|
|
435
|
+
if @allow_eopm && @range_decoder&.code&.zero? && @range_decoder&.stream
|
|
436
|
+
stream = @range_decoder.stream
|
|
590
437
|
# Try to peek at the next byte - if available, there's data AFTER EOPM
|
|
591
438
|
begin
|
|
592
439
|
next_byte = stream.getbyte
|
|
@@ -599,9 +446,9 @@ check_rc_finished: true)
|
|
|
599
446
|
rescue IOError, EOFError
|
|
600
447
|
# Stream doesn't support peeking or is exhausted, that's fine
|
|
601
448
|
end
|
|
602
|
-
elsif !@allow_eopm && @range_decoder&.
|
|
449
|
+
elsif !@allow_eopm && @range_decoder&.stream
|
|
603
450
|
# For LZMA2 mode (EOPM not allowed): check for leftover data
|
|
604
|
-
stream = @range_decoder.
|
|
451
|
+
stream = @range_decoder.stream
|
|
605
452
|
begin
|
|
606
453
|
next_byte = stream.getbyte
|
|
607
454
|
if next_byte
|
|
@@ -615,34 +462,14 @@ check_rc_finished: true)
|
|
|
615
462
|
end
|
|
616
463
|
end
|
|
617
464
|
|
|
618
|
-
#
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
#
|
|
622
|
-
if ENV["DEBUG_DICT_BUF"]
|
|
623
|
-
XzUtilsDecoderDebug.debug_puts "DEBUG: start_pos=#{start_pos}, @pos=#{@pos.inspect}, @dict_buf.size=#{@dict_buf.size}, LZ_DICT_INIT_POS=#{LZ_DICT_INIT_POS}"
|
|
624
|
-
end
|
|
625
|
-
valid_bytes = @dict_buf[start_pos...@pos]
|
|
626
|
-
# DEBUG: Show return value calculation
|
|
627
|
-
puts "DEBUG RETURN CALCULATION: call_num=#{call_num}, start_pos=#{start_pos}, @pos=#{@pos}, valid_bytes.size=#{@dict_buf[start_pos...@pos].size}, dict_full=#{@dict_full}, chunk_bytes_decoded=#{@chunk_bytes_decoded}" if ENV.fetch(
|
|
628
|
-
"LZMA_DEBUG", nil
|
|
629
|
-
) && @dict_full && @dict_full >= 220 && @dict_full <= 240 && call_num == 2
|
|
630
|
-
puts "DEBUG RETURN CALCULATION: call_num=#{call_num}, start_pos=#{start_pos}, @pos=#{@pos}, valid_bytes.size=#{@dict_buf[start_pos...@pos].size}, dict_full=#{@dict_full}, chunk_bytes_decoded=#{@chunk_bytes_decoded}" if ENV["LZMA_DEBUG"] && call_num == 2
|
|
631
|
-
# Filter out nil values (can happen during dictionary reset transitions)
|
|
632
|
-
valid_bytes = valid_bytes.map { |b| b.nil? ? 0 : b }
|
|
633
|
-
if ENV["DEBUG_DICT_BUF"]
|
|
634
|
-
XzUtilsDecoderDebug.debug_puts "DEBUG: valid_bytes=#{begin
|
|
635
|
-
valid_bytes.size
|
|
636
|
-
rescue StandardError
|
|
637
|
-
valid_bytes.inspect
|
|
638
|
-
end}"
|
|
639
|
-
end
|
|
640
|
-
valid_data = valid_bytes.pack("C*")
|
|
465
|
+
# Flush remaining output from the circular buffer
|
|
466
|
+
flush_circular_output(output)
|
|
467
|
+
|
|
468
|
+
# Return output
|
|
641
469
|
if output
|
|
642
|
-
|
|
643
|
-
valid_data.bytesize
|
|
470
|
+
@pos - start_pos
|
|
644
471
|
else
|
|
645
|
-
|
|
472
|
+
@output_accumulator.string.force_encoding(Encoding::BINARY)
|
|
646
473
|
end
|
|
647
474
|
end
|
|
648
475
|
|
|
@@ -660,23 +487,19 @@ check_rc_finished: true)
|
|
|
660
487
|
# @param preserve_dict [Boolean] If true, preserve dictionary state (pos, dict_full)
|
|
661
488
|
# @return [void]
|
|
662
489
|
def reset(new_lc: nil, new_lp: nil, new_pb: nil, preserve_dict: false)
|
|
663
|
-
if ENV["LZMA_DEBUG_RESET"]
|
|
664
|
-
warn "DEBUG reset[#{@decoder_id}] called: preserve_dict=#{preserve_dict}, @pos=#{@pos.inspect}, @dict_full=#{@dict_full.inspect}, @dict_buf.size=#{@dict_buf&.size || 'nil'}, nil_count=#{@dict_buf&.count(nil) || 'N/A'}"
|
|
665
|
-
end
|
|
666
|
-
|
|
667
|
-
# DEBUG: Trace reset calls around position 224-227
|
|
668
|
-
if @dict_full && @dict_full >= 220 && @dict_full <= 230
|
|
669
|
-
XzUtilsDecoderDebug.debug_puts "\n=== reset called at dict_full=#{@dict_full} ==="
|
|
670
|
-
XzUtilsDecoderDebug.debug_puts " preserve_dict=#{preserve_dict}"
|
|
671
|
-
XzUtilsDecoderDebug.debug_puts " Before reset: rep0/1/2/3=(#{@rep0},#{@rep1},#{@rep2},#{@rep3})"
|
|
672
|
-
end
|
|
673
|
-
|
|
674
490
|
# Update properties if provided
|
|
675
491
|
properties_changed = !!(new_lc || new_lp || new_pb)
|
|
676
492
|
@lc = new_lc if new_lc
|
|
677
493
|
@lp = new_lp if new_lp
|
|
678
494
|
@pb = new_pb if new_pb
|
|
679
495
|
|
|
496
|
+
# Recompute cached values when properties change
|
|
497
|
+
if properties_changed
|
|
498
|
+
@pb_mask = (1 << @pb) - 1
|
|
499
|
+
@pb_shift = 1 << @pb
|
|
500
|
+
@literal_mask = (0x100 << @lp) - (0x100 >> @lc)
|
|
501
|
+
end
|
|
502
|
+
|
|
680
503
|
# Reset state machine (XZ Utils line 1053)
|
|
681
504
|
# Always create a new state machine when resetting
|
|
682
505
|
@state = SdkStateMachine.new
|
|
@@ -695,53 +518,31 @@ check_rc_finished: true)
|
|
|
695
518
|
# - rc_reset sets range=UINT32_MAX, code=0, init_bytes_left=5
|
|
696
519
|
# - This MUST happen during reset, not deferred to decode_stream
|
|
697
520
|
# Reference: /Users/mulgogi/src/external/xz/src/liblzma/lzma/lzma_decoder.c:1061
|
|
698
|
-
|
|
699
|
-
@range_decoder.reset
|
|
700
|
-
if ENV["LZMA_DEBUG_RESET"]
|
|
701
|
-
warn "DEBUG reset[#{@decoder_id}]: Reset range_decoder, code=0x#{@range_decoder.code.to_s(16)}, init_bytes_remaining=#{@range_decoder.instance_variable_get(:@init_bytes_remaining)}"
|
|
702
|
-
end
|
|
703
|
-
end
|
|
521
|
+
@range_decoder&.reset
|
|
704
522
|
|
|
705
523
|
# Reinitialize probability models (XZ Utils lines 1064-1082)
|
|
706
524
|
# IMPORTANT: Use reset_models (reset in place) instead of init_models (create new)
|
|
707
525
|
# for state reset only. Only create new models when properties change.
|
|
708
526
|
if properties_changed
|
|
709
|
-
if ENV["LZMA_DEBUG_RESET"]
|
|
710
|
-
warn "DEBUG reset[#{@decoder_id}]: Properties changed, calling init_models (create new arrays)"
|
|
711
|
-
end
|
|
712
527
|
init_models
|
|
713
528
|
else
|
|
714
|
-
if ENV["LZMA_DEBUG_RESET"]
|
|
715
|
-
warn "DEBUG reset[#{@decoder_id}]: No properties changed, calling reset_models (reset in place)"
|
|
716
|
-
end
|
|
717
529
|
reset_models
|
|
718
530
|
end
|
|
719
531
|
|
|
720
532
|
# Reinitialize coders (needed for pb changes)
|
|
721
533
|
# Only recreate coders when properties have changed
|
|
722
|
-
if properties_changed
|
|
723
|
-
if ENV["LZMA_DEBUG_RESET"]
|
|
724
|
-
warn "DEBUG reset[#{@decoder_id}]: Properties changed, calling init_coders (create new coders)"
|
|
725
|
-
end
|
|
726
|
-
init_coders
|
|
727
|
-
elsif ENV["LZMA_DEBUG_RESET"]
|
|
728
|
-
warn "DEBUG reset[#{@decoder_id}]: No properties changed, skipping init_coders (preserve existing coders)"
|
|
729
|
-
end
|
|
534
|
+
init_coders if properties_changed
|
|
730
535
|
|
|
731
536
|
# Reset dictionary position and full count (XZ Utils pattern)
|
|
732
537
|
# Only reset if preserve_dict is false
|
|
733
538
|
unless preserve_dict
|
|
734
539
|
# Reinitialize dictionary buffer
|
|
735
540
|
# XZ Utils allocates a new buffer for each dictionary reset
|
|
736
|
-
@dict_buf =
|
|
541
|
+
@dict_buf = ("\0" * (@dict_size + LZ_DICT_INIT_POS)).b
|
|
737
542
|
@pos = LZ_DICT_INIT_POS
|
|
543
|
+
@dict_pos = LZ_DICT_INIT_POS
|
|
738
544
|
@dict_full = 0
|
|
739
545
|
@has_wrapped = false
|
|
740
|
-
if ENV["LZMA_DEBUG_RESET"]
|
|
741
|
-
warn "DEBUG reset after dict reset[#{@decoder_id}]: @pos=#{@pos}, @dict_full=#{@dict_full}, @dict_buf.size=#{@dict_buf.size}, @dict_buf.object_id=#{@dict_buf.object_id}, nil_count=#{@dict_buf.count(nil)}"
|
|
742
|
-
# Verify buffer initialization by checking a few positions
|
|
743
|
-
warn " Sample values: [576]=#{@dict_buf[576]}, [577]=#{@dict_buf[577]}, [578]=#{@dict_buf[578]}, [583]=#{@dict_buf[583]}"
|
|
744
|
-
end
|
|
745
546
|
end
|
|
746
547
|
|
|
747
548
|
nil
|
|
@@ -799,12 +600,6 @@ check_rc_finished: true)
|
|
|
799
600
|
#
|
|
800
601
|
# @return [void]
|
|
801
602
|
def prepare_state_reset
|
|
802
|
-
# DEBUG: Trace when prepare_state_reset is called
|
|
803
|
-
if ENV["LZMA_DEBUG"]
|
|
804
|
-
XzUtilsDecoderDebug.debug_puts "\n=== prepare_state_reset called (decoder_id=#{@decoder_id}) ==="
|
|
805
|
-
XzUtilsDecoderDebug.debug_puts " Before reset: rep0/1/2/3=(#{@rep0},#{@rep1},#{@rep2},#{@rep3})"
|
|
806
|
-
end
|
|
807
|
-
|
|
808
603
|
# Reset state machine (XZ Utils line 1053)
|
|
809
604
|
@state = SdkStateMachine.new
|
|
810
605
|
|
|
@@ -814,18 +609,9 @@ check_rc_finished: true)
|
|
|
814
609
|
@rep2 = 0
|
|
815
610
|
@rep3 = 0
|
|
816
611
|
|
|
817
|
-
# DEBUG: Show after reset
|
|
818
|
-
if ENV["LZMA_DEBUG"]
|
|
819
|
-
XzUtilsDecoderDebug.debug_puts " After reset: rep0/1/2/3=(#{@rep0},#{@rep1},#{@rep2},#{@rep3})"
|
|
820
|
-
end
|
|
821
|
-
|
|
822
612
|
# Reset probability models (XZ Utils init_temporals for control >= 0xA0)
|
|
823
613
|
reset_models
|
|
824
614
|
|
|
825
|
-
if ENV["LZMA_DEBUG"]
|
|
826
|
-
XzUtilsDecoderDebug.debug_puts "=== end prepare_state_reset (range decoder will be reset in finish_state_reset) ==="
|
|
827
|
-
end
|
|
828
|
-
|
|
829
615
|
nil
|
|
830
616
|
end
|
|
831
617
|
|
|
@@ -837,23 +623,12 @@ check_rc_finished: true)
|
|
|
837
623
|
#
|
|
838
624
|
# @return [void]
|
|
839
625
|
def reset_state_machine_only
|
|
840
|
-
# DEBUG: Trace when reset_state_machine_only is called
|
|
841
|
-
if @dict_full && @dict_full >= 220 && @dict_full <= 230
|
|
842
|
-
XzUtilsDecoderDebug.debug_puts "\n=== reset_state_machine_only called at dict_full=#{@dict_full} (decoder_id=#{@decoder_id}) ==="
|
|
843
|
-
XzUtilsDecoderDebug.debug_puts " Before reset: rep0/1/2/3=(#{@rep0},#{@rep1},#{@rep2},#{@rep3})"
|
|
844
|
-
end
|
|
845
|
-
|
|
846
626
|
# Reset state machine only (XZ Utils line 1053)
|
|
847
627
|
@state = SdkStateMachine.new
|
|
848
628
|
|
|
849
629
|
# Reset probability models (XZ Utils init_temporals for control >= 0xA0)
|
|
850
630
|
reset_models
|
|
851
631
|
|
|
852
|
-
# DEBUG: Show after reset (note: rep distances are preserved)
|
|
853
|
-
if @dict_full && @dict_full >= 220 && @dict_full <= 230
|
|
854
|
-
XzUtilsDecoderDebug.debug_puts " After reset: rep0/1/2/3=(#{@rep0},#{@rep1},#{@rep2},#{@rep3}) (preserved)"
|
|
855
|
-
end
|
|
856
|
-
|
|
857
632
|
nil
|
|
858
633
|
end
|
|
859
634
|
|
|
@@ -872,30 +647,7 @@ check_rc_finished: true)
|
|
|
872
647
|
# Reset range decoder (XZ Utils rc_reset)
|
|
873
648
|
# This reinitializes the range decoder for the new chunk
|
|
874
649
|
# The reset will read 5 bytes from the input when decode_stream starts
|
|
875
|
-
|
|
876
|
-
if ENV["LZMA_DEBUG"]
|
|
877
|
-
input_pos = begin
|
|
878
|
-
@input.pos
|
|
879
|
-
rescue StandardError
|
|
880
|
-
"N/A"
|
|
881
|
-
end
|
|
882
|
-
input_size = begin
|
|
883
|
-
@input.size
|
|
884
|
-
rescue StandardError
|
|
885
|
-
"N/A"
|
|
886
|
-
end
|
|
887
|
-
XzUtilsDecoderDebug.debug_puts "=== finish_state_reset: resetting range_decoder, input pos=#{input_pos}, size=#{input_size}"
|
|
888
|
-
end
|
|
889
|
-
@range_decoder.reset
|
|
890
|
-
if ENV["LZMA_DEBUG"]
|
|
891
|
-
input_pos_after = begin
|
|
892
|
-
@input.pos
|
|
893
|
-
rescue StandardError
|
|
894
|
-
"N/A"
|
|
895
|
-
end
|
|
896
|
-
XzUtilsDecoderDebug.debug_puts "=== finish_state_reset: after reset, input pos=#{input_pos_after}, range_decoder.code=0x#{@range_decoder.code.to_s(16)}"
|
|
897
|
-
end
|
|
898
|
-
end
|
|
650
|
+
@range_decoder&.reset
|
|
899
651
|
end
|
|
900
652
|
|
|
901
653
|
def reset_state_only
|
|
@@ -931,45 +683,13 @@ check_rc_finished: true)
|
|
|
931
683
|
def set_input(new_input)
|
|
932
684
|
@input = new_input
|
|
933
685
|
|
|
934
|
-
# DEBUG: Trace input stream contents
|
|
935
|
-
if ENV.fetch("LZMA_DEBUG",
|
|
936
|
-
nil) && @dict_full && @dict_full >= 220 && @dict_full <= 230
|
|
937
|
-
puts "\n=== set_input at dict_full=#{@dict_full} ==="
|
|
938
|
-
puts " new_input.size=#{new_input.size}"
|
|
939
|
-
puts " new_input.pos=#{new_input.pos}"
|
|
940
|
-
puts " new_input.class=#{new_input.class}"
|
|
941
|
-
|
|
942
|
-
# Read first 10 bytes manually
|
|
943
|
-
first_bytes = []
|
|
944
|
-
10.times do |_i|
|
|
945
|
-
byte = new_input.getbyte
|
|
946
|
-
break if byte.nil?
|
|
947
|
-
|
|
948
|
-
first_bytes << byte
|
|
949
|
-
end
|
|
950
|
-
puts " First 10 bytes: #{first_bytes.map do |b|
|
|
951
|
-
"0x#{b.to_s(16).upcase}"
|
|
952
|
-
end.join(' ')}"
|
|
953
|
-
|
|
954
|
-
new_input.rewind
|
|
955
|
-
test_byte = new_input.getbyte
|
|
956
|
-
puts " Test getbyte: 0x#{test_byte.to_s(16).upcase}" if test_byte
|
|
957
|
-
new_input.rewind
|
|
958
|
-
end
|
|
959
|
-
|
|
960
686
|
# Create range decoder if it doesn't exist (first chunk)
|
|
961
687
|
if @range_decoder.nil?
|
|
962
688
|
@range_decoder = RangeDecoder.new(@input)
|
|
963
|
-
if ENV["LZMA_DEBUG"]
|
|
964
|
-
XzUtilsDecoderDebug.debug_puts "=== set_input: created NEW range_decoder, input has #{@input.size} bytes"
|
|
965
|
-
end
|
|
966
689
|
else
|
|
967
690
|
# Update the range decoder's input stream to the new input
|
|
968
691
|
# This is needed because RangeDecoder holds a reference to the stream
|
|
969
692
|
@range_decoder.update_stream(@input)
|
|
970
|
-
if ENV["LZMA_DEBUG"]
|
|
971
|
-
XzUtilsDecoderDebug.debug_puts "=== set_input: reusing range_decoder, new input has #{@input.size} bytes, pos=#{@input.pos}"
|
|
972
|
-
end
|
|
973
693
|
end
|
|
974
694
|
end
|
|
975
695
|
|
|
@@ -985,29 +705,30 @@ check_rc_finished: true)
|
|
|
985
705
|
# @param data [String] Uncompressed data to add to dictionary
|
|
986
706
|
# @return [void]
|
|
987
707
|
def add_to_dictionary(data)
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
XzUtilsDecoderDebug.debug_puts "=== add_to_dictionary: adding #{data.bytesize} bytes to dictionary[#{@decoder_id}], current dict_full=#{@dict_full}, pos=#{@pos}"
|
|
991
|
-
end
|
|
992
|
-
|
|
708
|
+
idx = @dict_pos
|
|
709
|
+
buf_end = @buf_end
|
|
993
710
|
data.each_byte do |byte|
|
|
994
|
-
@dict_buf
|
|
711
|
+
@dict_buf.setbyte(idx, byte)
|
|
995
712
|
@pos += 1
|
|
713
|
+
idx += 1
|
|
714
|
+
idx = LZ_DICT_INIT_POS if idx >= buf_end
|
|
996
715
|
end
|
|
716
|
+
@dict_pos = idx
|
|
997
717
|
|
|
998
718
|
# Update dict_full to reflect new data
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
if ENV["LZMA_DEBUG"]
|
|
1007
|
-
XzUtilsDecoderDebug.debug_puts "=== add_to_dictionary: after adding, dict_full=#{@dict_full} (was #{old_dict_full}), pos=#{@pos}"
|
|
719
|
+
unless @has_wrapped
|
|
720
|
+
@dict_full = @pos - LZ_DICT_INIT_POS
|
|
721
|
+
if @dict_full >= @dict_size
|
|
722
|
+
@has_wrapped = true
|
|
723
|
+
@dict_full = @dict_size
|
|
724
|
+
end
|
|
1008
725
|
end
|
|
1009
726
|
end
|
|
1010
727
|
|
|
728
|
+
# No-op: circular buffer has fixed size, no compaction needed.
|
|
729
|
+
# Kept for API compatibility — LZMA2 decoder calls this between chunks.
|
|
730
|
+
def compact_buffer; end
|
|
731
|
+
|
|
1011
732
|
# Set uncompressed size for chunked decoding
|
|
1012
733
|
#
|
|
1013
734
|
# XZ Utils pattern (lzma2_decoder.c:140-141):
|
|
@@ -1020,10 +741,6 @@ check_rc_finished: true)
|
|
|
1020
741
|
def set_uncompressed_size(size, allow_eopm: true)
|
|
1021
742
|
@uncompressed_size = size
|
|
1022
743
|
@allow_eopm = allow_eopm
|
|
1023
|
-
# DEBUG: Track when uncompressed size is set
|
|
1024
|
-
if ENV["LZMA_DEBUG"]
|
|
1025
|
-
puts "DEBUG set_uncompressed_size: size=#{size}, @decoder_id=#{@decoder_id}, @dict_full=#{@dict_full}"
|
|
1026
|
-
end
|
|
1027
744
|
end
|
|
1028
745
|
|
|
1029
746
|
private
|
|
@@ -1136,13 +853,6 @@ check_rc_finished: true)
|
|
|
1136
853
|
@rep_length_coder = LengthCoder.new(pos_states)
|
|
1137
854
|
@distance_coder = DistanceCoder.new(NUM_LEN_TO_POS_STATES)
|
|
1138
855
|
|
|
1139
|
-
if ENV["TRACE_MODEL_INIT"]
|
|
1140
|
-
puts "[XzUtilsDecoder.init] slot_encoders len_state=0 object_id=#{@distance_coder.instance_variable_get(:@slot_encoders)[0].object_id}"
|
|
1141
|
-
puts "[XzUtilsDecoder.init] slot_encoders[0][1] object_id=#{@distance_coder.instance_variable_get(:@slot_encoders)[0][1].object_id}"
|
|
1142
|
-
puts "[XzUtilsDecoder.init] is_match_models object_id=#{@is_match_models.object_id}"
|
|
1143
|
-
puts "[XzUtilsDecoder.init] is_match_models[0] object_id=#{@is_match_models[0].object_id}"
|
|
1144
|
-
end
|
|
1145
|
-
|
|
1146
856
|
# Update probability model indices to match new pos_states
|
|
1147
857
|
# This is critical when pb changes between chunks
|
|
1148
858
|
@pos_states = pos_states
|
|
@@ -1174,202 +884,36 @@ check_rc_finished: true)
|
|
|
1174
884
|
#
|
|
1175
885
|
# @return [void]
|
|
1176
886
|
def decode_literal
|
|
1177
|
-
# DEBUG: Trace literals around position 224-227
|
|
1178
|
-
old_dict_full = @dict_full
|
|
1179
|
-
|
|
1180
|
-
# DEBUG: Track how many times we're called for each position
|
|
1181
|
-
if ENV["LZMA_DEBUG_DECODE_LITERAL"]
|
|
1182
|
-
caller_info = caller(1..1).first
|
|
1183
|
-
XzUtilsDecoderDebug.debug_puts "DEBUG decode_literal[#{@decoder_id}]: pos=#{@pos}, dict_full=#{@dict_full}, from=#{caller_info.label}"
|
|
1184
|
-
end
|
|
1185
|
-
|
|
1186
|
-
# DEBUG: Check array integrity before decode
|
|
1187
|
-
if ENV.fetch("LZMA_DEBUG_ARRAY",
|
|
1188
|
-
nil) && @dict_full.positive? && @pos > 1
|
|
1189
|
-
idx = @pos - 1
|
|
1190
|
-
if @dict_buf[idx].nil?
|
|
1191
|
-
raise "DEBUG before decode: @dict_buf[#{idx}] is nil! @pos=#{@pos}, @dict_full=#{@dict_full}, @dict_buf.size=#{@dict_buf.size}, nil_count=#{@dict_buf.count(nil)}"
|
|
1192
|
-
end
|
|
1193
|
-
end
|
|
1194
|
-
|
|
1195
887
|
# Calculate literal state using SDK formula
|
|
1196
888
|
lit_state = calculate_literal_state
|
|
1197
889
|
|
|
1198
|
-
# DEBUG: Trace lit_state at position 61
|
|
1199
|
-
if @dict_full == 61 && ENV["TRACE_LITERAL_61"]
|
|
1200
|
-
XzUtilsDecoderDebug.debug_puts "=== CALC_LITERAL_STATE at dict_full=61 ==="
|
|
1201
|
-
XzUtilsDecoderDebug.debug_puts " prev_byte=#{@dict_full.positive? ? @dict_buf[@pos - 1] : 0}"
|
|
1202
|
-
XzUtilsDecoderDebug.debug_puts " lit_state=#{lit_state}"
|
|
1203
|
-
XzUtilsDecoderDebug.debug_puts " lc=#{@lc}, lp=#{@lp}"
|
|
1204
|
-
XzUtilsDecoderDebug.debug_puts " state.value=#{@state.value}"
|
|
1205
|
-
XzUtilsDecoderDebug.debug_puts " use_matched_literal?=#{@state.use_matched_literal?}"
|
|
1206
|
-
XzUtilsDecoderDebug.debug_puts " dict_full.positive?=#{@dict_full.positive?}"
|
|
1207
|
-
puts
|
|
1208
|
-
end
|
|
1209
|
-
|
|
1210
890
|
# Decode literal (matched or unmatched)
|
|
1211
891
|
# Check if dictionary has any valid bytes (XZ Utils: dict->full > 0)
|
|
1212
892
|
if @state.use_matched_literal? && @dict_full.positive?
|
|
1213
|
-
# DEBUG: Track which branch is taken
|
|
1214
|
-
if @dict_full == 61 && ENV["TRACE_LITERAL_61"]
|
|
1215
|
-
XzUtilsDecoderDebug.debug_puts " TAKING MATCHED LITERAL PATH"
|
|
1216
|
-
XzUtilsDecoderDebug.debug_puts " rep0=#{@rep0}"
|
|
1217
|
-
match_byte_pos_calc = LZ_DICT_INIT_POS + @dict_full - @rep0 - 1
|
|
1218
|
-
XzUtilsDecoderDebug.debug_puts " match_byte_pos (calc)=#{match_byte_pos_calc}"
|
|
1219
|
-
puts
|
|
1220
|
-
end
|
|
1221
|
-
|
|
1222
893
|
# Matched literal: use match byte from dictionary at distance rep0
|
|
1223
894
|
# XZ Utils dict_get pattern: dict->buf[dict->pos - distance - 1]
|
|
1224
|
-
|
|
1225
|
-
# not the buffer position with offset!
|
|
1226
|
-
# omnizip uses @pos for buffer position (includes LZ_DICT_INIT_POS offset)
|
|
1227
|
-
# and @dict_full for actual output position (starts at 0)
|
|
1228
|
-
# So we must convert: buffer_pos = LZ_DICT_INIT_POS + (output_pos - rep0 - 1)
|
|
1229
|
-
match_byte_pos = LZ_DICT_INIT_POS + @dict_full - @rep0 - 1
|
|
1230
|
-
match_byte = @dict_buf[match_byte_pos]
|
|
1231
|
-
if ENV["LZMA_DEBUG"]
|
|
1232
|
-
warn "DEBUG: matched literal - dict_full=#{@dict_full}, rep0=#{@rep0}, reading dict_buf[#{match_byte_pos}]=0x#{match_byte.to_s(16).upcase} ('#{match_byte.chr}'), lit_state=#{lit_state}, state=#{@state.value}"
|
|
1233
|
-
end
|
|
895
|
+
match_byte = @dict_buf.getbyte(dict_index(@pos - @rep0 - 1))
|
|
1234
896
|
byte = @literal_decoder.decode_matched(match_byte, lit_state, @lc,
|
|
1235
897
|
@range_decoder, @literal_models)
|
|
1236
|
-
|
|
1237
|
-
# DEBUG: Trace decoded byte at position 61
|
|
1238
|
-
if @dict_full == 61 && ENV["TRACE_LITERAL_61"]
|
|
1239
|
-
XzUtilsDecoderDebug.debug_puts " DECODED MATCHED LITERAL: 0x#{byte.to_s(16).upcase} ('#{byte.chr}')"
|
|
1240
|
-
XzUtilsDecoderDebug.debug_puts " match_byte=0x#{match_byte.to_s(16).upcase} ('#{match_byte.chr}')"
|
|
1241
|
-
puts
|
|
1242
|
-
end
|
|
1243
898
|
else
|
|
1244
899
|
# Unmatched literal: simple 8-bit decoding
|
|
1245
|
-
if @dict_full == 61 && ENV["TRACE_LITERAL_61"]
|
|
1246
|
-
XzUtilsDecoderDebug.debug_puts " TAKING UNMATCHED LITERAL PATH"
|
|
1247
|
-
puts
|
|
1248
|
-
end
|
|
1249
|
-
|
|
1250
|
-
if ENV["LZMA_DEBUG"]
|
|
1251
|
-
warn "DEBUG: calling decode_unmatched - pos=#{@pos}, lit_state=#{lit_state}"
|
|
1252
|
-
end
|
|
1253
900
|
byte = @literal_decoder.decode_unmatched(lit_state, @lc,
|
|
1254
901
|
@range_decoder, @literal_models)
|
|
1255
902
|
end
|
|
1256
903
|
|
|
1257
|
-
if ENV["LZMA_DEBUG"]
|
|
1258
|
-
warn "DEBUG: decode_literal RETURNED - pos=#{@pos}, byte=#{'0x%02X' % byte} ('#{byte.chr}'), lit_state=#{lit_state}"
|
|
1259
|
-
end
|
|
1260
|
-
|
|
1261
|
-
# DEBUG: Check byte value at critical positions
|
|
1262
|
-
if @dict_full == 256
|
|
1263
|
-
XzUtilsDecoderDebug.debug_puts "DEBUG: About to store 257th byte (pos=#{@pos}, dict_full=#{@dict_full})"
|
|
1264
|
-
XzUtilsDecoderDebug.debug_puts " byte.class=#{byte.class}"
|
|
1265
|
-
XzUtilsDecoderDebug.debug_puts " byte=#{byte.inspect}"
|
|
1266
|
-
XzUtilsDecoderDebug.debug_puts " byte.is_a?(Integer)=#{byte.is_a?(Integer)}"
|
|
1267
|
-
if byte.is_a?(Integer)
|
|
1268
|
-
XzUtilsDecoderDebug.debug_puts " byte value=#{byte}"
|
|
1269
|
-
XzUtilsDecoderDebug.debug_puts " Expected byte value=0"
|
|
1270
|
-
else
|
|
1271
|
-
XzUtilsDecoderDebug.debug_puts " byte is not an Integer!"
|
|
1272
|
-
XzUtilsDecoderDebug.debug_puts " byte.ord=#{byte.ord}"
|
|
1273
|
-
end
|
|
1274
|
-
end
|
|
1275
|
-
|
|
1276
|
-
if ENV["LZMA_DEBUG"]
|
|
1277
|
-
decoded_bytes = @dict_full.positive? ? @dict_buf[LZ_DICT_INIT_POS..].map(&:chr).join : ""
|
|
1278
|
-
warn "DEBUG: decode_literal - pos=#{@pos}, byte=#{'0x%02X' % byte} ('#{byte.chr}'), state=#{@state.value}, dict_full=#{@dict_full}, decoded_so_far='#{decoded_bytes[-10..]}'"
|
|
1279
|
-
end
|
|
1280
|
-
|
|
1281
|
-
# DEBUG: Detailed trace around position 256
|
|
1282
|
-
if ENV["LZMA_DEBUG"] && @dict_full.between?(230, 265)
|
|
1283
|
-
expected = @dict_full % 256
|
|
1284
|
-
match = byte == expected ? "✓" : "✗ MISMATCH!"
|
|
1285
|
-
XzUtilsDecoderDebug.debug_puts " [LITERAL] dict_full=#{@dict_full}: 0x#{byte.to_s(16).upcase} (expected 0x#{expected.to_s(16).upcase}) #{match}"
|
|
1286
|
-
if @dict_full == 233
|
|
1287
|
-
XzUtilsDecoderDebug.debug_puts " DETAILED TRACE at dict_full=233 (pos=#{@pos}):"
|
|
1288
|
-
XzUtilsDecoderDebug.debug_puts " byte=0x#{byte.to_s(16)} ('#{begin
|
|
1289
|
-
byte.chr
|
|
1290
|
-
rescue StandardError
|
|
1291
|
-
'?'
|
|
1292
|
-
end}')"
|
|
1293
|
-
XzUtilsDecoderDebug.debug_puts " state.value=#{@state.value}, lit_state=#{lit_state}"
|
|
1294
|
-
XzUtilsDecoderDebug.debug_puts " use_matched_literal?=#{@state.use_matched_literal?}"
|
|
1295
|
-
prev_byte_val = @dict_full.positive? ? @dict_buf[@pos - 1] : "N/A"
|
|
1296
|
-
XzUtilsDecoderDebug.debug_puts " prev_byte=#{prev_byte_val.inspect} (#{if prev_byte_val.is_a?(Integer)
|
|
1297
|
-
"0x#{prev_byte_val.to_s(16)} ('#{begin
|
|
1298
|
-
prev_byte_val.chr
|
|
1299
|
-
rescue StandardError
|
|
1300
|
-
'?'
|
|
1301
|
-
end}')"
|
|
1302
|
-
else
|
|
1303
|
-
'N/A'
|
|
1304
|
-
end})"
|
|
1305
|
-
XzUtilsDecoderDebug.debug_puts " range_decoder.range=0x#{@range_decoder.range.to_s(16)}, range_decoder.code=0x#{@range_decoder.code.to_s(16)}"
|
|
1306
|
-
XzUtilsDecoderDebug.debug_puts " input.pos=#{@input.pos}, input.size=#{@input.size}"
|
|
1307
|
-
end
|
|
1308
|
-
if @dict_full == 256
|
|
1309
|
-
XzUtilsDecoderDebug.debug_puts " pos=#{@pos}, lit_state=#{lit_state}, state.value=#{@state.value}"
|
|
1310
|
-
XzUtilsDecoderDebug.debug_puts " use_matched_literal?=#{@state.use_matched_literal?}"
|
|
1311
|
-
end
|
|
1312
|
-
end
|
|
1313
|
-
|
|
1314
904
|
# Update state and dictionary
|
|
1315
905
|
# XZ Utils dict_put pattern:
|
|
1316
906
|
# dict->buf[dict->pos++] = byte;
|
|
1317
907
|
# if (!dict->has_wrapped)
|
|
1318
908
|
# dict->full = dict->pos - LZ_DICT_INIT_POS;
|
|
1319
909
|
@state.update_literal
|
|
1320
|
-
warn "DEBUG: After update_literal - state=#{@state.value}" if ENV["LZMA_DEBUG"]
|
|
1321
910
|
|
|
1322
911
|
# Write to dictionary buffer at current position
|
|
1323
912
|
# XZ Utils dict_put pattern: dict->buf[dict->pos++] = byte;
|
|
1324
|
-
|
|
1325
|
-
if @pos == 576 + 256
|
|
1326
|
-
XzUtilsDecoderDebug.debug_puts "DEBUG: Storing byte at pos 832 (256th decoded byte)"
|
|
1327
|
-
XzUtilsDecoderDebug.debug_puts " byte.class=#{byte.class}"
|
|
1328
|
-
XzUtilsDecoderDebug.debug_puts " byte=#{byte}"
|
|
1329
|
-
XzUtilsDecoderDebug.debug_puts " byte.ord=#{byte.is_a?(String) ? byte.ord : 'N/A (not a string)'}"
|
|
1330
|
-
XzUtilsDecoderDebug.debug_puts " Integer value=#{byte.is_a?(Integer) ? byte : byte.ord}"
|
|
1331
|
-
end
|
|
1332
|
-
@dict_buf[@pos] = byte
|
|
1333
|
-
# DEBUG: Track array size changes
|
|
1334
|
-
if ENV["LZMA_DEBUG_ARRAY_WRITE"] && @dict_buf.size != (@dict_size + LZ_DICT_INIT_POS)
|
|
1335
|
-
XzUtilsDecoderDebug.debug_puts "DEBUG: Array expanded! pos=#{@pos}, byte=#{byte}, old_size=#{@dict_buf.size - 1}, new_size=#{@dict_buf.size}, decoder_id=#{@decoder_id}"
|
|
1336
|
-
XzUtilsDecoderDebug.debug_puts " Writing beyond original size caused expansion!"
|
|
1337
|
-
end
|
|
1338
|
-
if ENV["LZMA_DEBUG_ARRAY_WRITE"]
|
|
1339
|
-
XzUtilsDecoderDebug.debug_puts "DEBUG write[#{@decoder_id}]: pos=#{@pos}, byte=#{byte}, dict_buf.size=#{@dict_buf.size}, dict_buf.object_id=#{@dict_buf.object_id}, nil_count=#{@dict_buf.count(nil)}"
|
|
1340
|
-
end
|
|
1341
|
-
if ENV["LZMA_DEBUG_ARRAY"]
|
|
1342
|
-
# Verify the write succeeded
|
|
1343
|
-
if @dict_buf[@pos] != byte
|
|
1344
|
-
raise "DEBUG after write: @dict_buf[#{@pos}] = #{@dict_buf[@pos].inspect}, expected #{byte}! nil_count=#{@dict_buf.count(nil)}"
|
|
1345
|
-
end
|
|
1346
|
-
if @dict_full.positive? && @pos > LZ_DICT_INIT_POS && @dict_buf[@pos - 1].nil?
|
|
1347
|
-
raise "DEBUG after write: @dict_buf[#{@pos - 1}] is nil! @pos=#{@pos}, @dict_full=#{@dict_full}"
|
|
1348
|
-
end
|
|
1349
|
-
end
|
|
913
|
+
@dict_buf.setbyte(@dict_pos, byte)
|
|
1350
914
|
@pos += 1
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
if ENV["TRACE_ARM64_BYTES"]
|
|
1354
|
-
@arm64_trace ||= []
|
|
1355
|
-
if @arm64_trace.size < 20
|
|
1356
|
-
@arm64_trace << [@dict_full, @pos, byte.class,
|
|
1357
|
-
byte.is_a?(Integer) ? byte : byte.ord, @dict_buf[@pos]]
|
|
1358
|
-
if @arm64_trace.size == 20
|
|
1359
|
-
# Dump the trace
|
|
1360
|
-
puts "\n=== ARM64 BYTE TRACE (first 20 bytes) ==="
|
|
1361
|
-
puts "Decoder ID: #{@decoder_id}"
|
|
1362
|
-
@arm64_trace.each_with_index do |entry, i|
|
|
1363
|
-
df, p, _, val, stored = entry
|
|
1364
|
-
puts " [#{i + 1}] dict_full=#{df.to_s.rjust(6)}, pos=#{p.to_s.rjust(6)}, byte=#{val.to_s.rjust(3)} (0x#{val.to_s(16).upcase.rjust(
|
|
1365
|
-
2, '0'
|
|
1366
|
-
)}) stored=#{stored.inspect}"
|
|
1367
|
-
end
|
|
1368
|
-
puts "=========================================\n"
|
|
1369
|
-
$stderr.flush
|
|
1370
|
-
end
|
|
1371
|
-
end
|
|
1372
|
-
end
|
|
915
|
+
@dict_pos += 1
|
|
916
|
+
@dict_pos = LZ_DICT_INIT_POS if @dict_pos >= @buf_end
|
|
1373
917
|
|
|
1374
918
|
# Update dict_full (XZ Utils pattern)
|
|
1375
919
|
# When dict_full reaches dict_size, the dictionary is full
|
|
@@ -1390,13 +934,6 @@ check_rc_finished: true)
|
|
|
1390
934
|
if @uncompressed_size != 0xFFFFFFFFFFFFFFFF
|
|
1391
935
|
@chunk_bytes_decoded += 1
|
|
1392
936
|
end
|
|
1393
|
-
|
|
1394
|
-
# DEBUG: Show literal decode for position 220-230
|
|
1395
|
-
if old_dict_full.between?(220, 230)
|
|
1396
|
-
XzUtilsDecoderDebug.debug_puts "\n=== decode_literal at dict_full=#{old_dict_full} ==="
|
|
1397
|
-
XzUtilsDecoderDebug.debug_puts " Decoded: 0x#{byte.to_s(16)} ('#{byte.chr}')"
|
|
1398
|
-
XzUtilsDecoderDebug.debug_puts " rep0/1/2/3=(#{@rep0},#{@rep1},#{@rep2},#{@rep3})"
|
|
1399
|
-
end
|
|
1400
937
|
end
|
|
1401
938
|
|
|
1402
939
|
# Decode a match
|
|
@@ -1416,60 +953,18 @@ check_rc_finished: true)
|
|
|
1416
953
|
#
|
|
1417
954
|
# @return [Boolean] True if EOS marker detected, false otherwise
|
|
1418
955
|
def decode_match
|
|
1419
|
-
pos_state = @pos &
|
|
1420
|
-
|
|
1421
|
-
# DEEP DEBUG: Trace every detail at position 227
|
|
1422
|
-
if ENV["LZMA_DEBUG"] && @dict_full == 227
|
|
1423
|
-
puts "\n=== DEEP DEBUG at dict_full=227 ==="
|
|
1424
|
-
puts " State: #{@state.value}, pos_state=#{pos_state}"
|
|
1425
|
-
puts " Rep distances BEFORE: (#{@rep0},#{@rep1},#{@rep2},#{@rep3})"
|
|
1426
|
-
puts " Range decoder: range=0x#{@range_decoder.range.to_s(16).upcase}, code=0x#{@range_decoder.code.to_s(16).upcase}"
|
|
1427
|
-
input_buffer = @range_decoder.instance_variable_get(:@input)
|
|
1428
|
-
puts " Input buffer: #{input_buffer ? input_buffer.size : 'nil'} bytes"
|
|
1429
|
-
end
|
|
956
|
+
pos_state = @pos & @pb_mask
|
|
1430
957
|
|
|
1431
958
|
# Decode is_rep bit
|
|
1432
959
|
is_rep_model = @is_rep_models[@state.value]
|
|
1433
|
-
if ENV["TRACE_IS_REP"]
|
|
1434
|
-
range_val = @range_decoder.range
|
|
1435
|
-
code_val = @range_decoder.code
|
|
1436
|
-
puts "[XzUtilsDecoder.decode_match] Before is_rep: state.value=#{@state.value}"
|
|
1437
|
-
puts " is_rep_model.object_id=#{is_rep_model.object_id}, prob=#{is_rep_model.probability}"
|
|
1438
|
-
puts " range=#{range_val} (0x#{range_val.to_s(16)}), code=#{code_val} (0x#{code_val.to_s(16)})"
|
|
1439
|
-
bound_calc = (range_val >> 11) * is_rep_model.probability
|
|
1440
|
-
puts " bound=(#{range_val} >> 11) * #{is_rep_model.probability} = #{bound_calc} (0x#{bound_calc.to_s(16)})"
|
|
1441
|
-
puts " code < bound? #{code_val < bound_calc}"
|
|
1442
|
-
end
|
|
1443
960
|
is_rep = @range_decoder.decode_bit(is_rep_model)
|
|
1444
961
|
|
|
1445
|
-
if ENV["TRACE_IS_REP"]
|
|
1446
|
-
range_val = @range_decoder.range
|
|
1447
|
-
code_val = @range_decoder.code
|
|
1448
|
-
puts "[XzUtilsDecoder.decode_match] Decoded is_rep=#{is_rep} with prob=#{is_rep_model.probability}"
|
|
1449
|
-
puts " After is_rep: range=#{range_val} (0x#{range_val.to_s(16)}), code=#{code_val} (0x#{code_val.to_s(16)})"
|
|
1450
|
-
end
|
|
1451
|
-
|
|
1452
|
-
if ENV["LZMA_DEBUG"] && @dict_full == 227
|
|
1453
|
-
puts " Decoded is_rep bit: #{is_rep} (#{@is_rep_models[@state.value].probability})"
|
|
1454
|
-
puts " After is_rep: range=0x#{@range_decoder.range.to_s(16).upcase}, code=0x#{@range_decoder.code.to_s(16).upcase}"
|
|
1455
|
-
end
|
|
1456
|
-
|
|
1457
|
-
if ENV["LZMA_DEBUG"]
|
|
1458
|
-
warn "DEBUG: decode_match START - is_rep=#{is_rep}, state.value=#{@state.value}, pos_state=#{pos_state}, rep0/1/2/3=(#{@rep0},#{@rep1},#{@rep2},#{@rep3})"
|
|
1459
|
-
end
|
|
1460
|
-
|
|
1461
962
|
if is_rep.zero?
|
|
1462
963
|
# Regular match (not rep)
|
|
1463
|
-
if ENV["LZMA_DEBUG"] && @dict_full.between?(220, 240)
|
|
1464
|
-
puts "DEBUG pos #{@dict_full}: Regular match (not rep)"
|
|
1465
|
-
end
|
|
1466
964
|
# Return result from decode_regular_match (true if EOS marker detected)
|
|
1467
965
|
return true if decode_regular_match(pos_state)
|
|
1468
966
|
else
|
|
1469
967
|
# Rep match - decode which rep distance to use
|
|
1470
|
-
if ENV["LZMA_DEBUG"] && @dict_full.between?(220, 240)
|
|
1471
|
-
puts "DEBUG pos #{@dict_full}: Rep match (is_rep=1)"
|
|
1472
|
-
end
|
|
1473
968
|
decode_rep_match(pos_state)
|
|
1474
969
|
end
|
|
1475
970
|
|
|
@@ -1488,14 +983,8 @@ check_rc_finished: true)
|
|
|
1488
983
|
# @param pos_state [Integer] Position state
|
|
1489
984
|
# @return [Boolean] True if EOS marker detected, false otherwise
|
|
1490
985
|
def decode_regular_match(pos_state)
|
|
1491
|
-
# DEBUG: Trace matches around dict_full = 60-63
|
|
1492
|
-
old_dict_full = @dict_full
|
|
1493
|
-
old_rep0 = @rep0
|
|
1494
|
-
old_state = @state.value
|
|
1495
|
-
|
|
1496
986
|
# Decode match length
|
|
1497
|
-
length_encoded = @length_coder.decode(@range_decoder,
|
|
1498
|
-
pos_state)
|
|
987
|
+
length_encoded = @length_coder.decode(@range_decoder, pos_state)
|
|
1499
988
|
length = length_encoded + MATCH_LEN_MIN
|
|
1500
989
|
|
|
1501
990
|
# Calculate length state for distance decoding
|
|
@@ -1508,54 +997,11 @@ check_rc_finished: true)
|
|
|
1508
997
|
NUM_LEN_TO_POS_STATES - 1
|
|
1509
998
|
end
|
|
1510
999
|
|
|
1511
|
-
# DEBUG: Show bytes being copied
|
|
1512
|
-
if old_dict_full.between?(210, 230) || ENV["LZMA_DEBUG_DISTANCE"]
|
|
1513
|
-
XzUtilsDecoderDebug.debug_puts "\n=== decode_regular_match at dict_full=#{old_dict_full} ===" if old_dict_full.between?(
|
|
1514
|
-
210, 230
|
|
1515
|
-
)
|
|
1516
|
-
puts "[DISTANCE_DECODER] decode_regular_match at dict_full=#{old_dict_full}" if ENV["LZMA_DEBUG_DISTANCE"]
|
|
1517
|
-
XzUtilsDecoderDebug.debug_puts " pos_state=#{pos_state}" if old_dict_full.between?(
|
|
1518
|
-
210, 230
|
|
1519
|
-
)
|
|
1520
|
-
puts "[DISTANCE_DECODER] pos_state=#{pos_state}" if ENV["LZMA_DEBUG_DISTANCE"]
|
|
1521
|
-
XzUtilsDecoderDebug.debug_puts " state=#{old_state}" if old_dict_full.between?(
|
|
1522
|
-
210, 230
|
|
1523
|
-
)
|
|
1524
|
-
puts "[DISTANCE_DECODER] state=#{old_state}" if ENV["LZMA_DEBUG_DISTANCE"]
|
|
1525
|
-
XzUtilsDecoderDebug.debug_puts " length_encoded=#{length_encoded} length=#{length}" if old_dict_full.between?(
|
|
1526
|
-
210, 230
|
|
1527
|
-
)
|
|
1528
|
-
puts "[DISTANCE_DECODER] length_encoded=#{length_encoded} length=#{length}" if ENV["LZMA_DEBUG_DISTANCE"]
|
|
1529
|
-
XzUtilsDecoderDebug.debug_puts " len_state=#{len_state}" if old_dict_full.between?(
|
|
1530
|
-
210, 230
|
|
1531
|
-
)
|
|
1532
|
-
puts "[DISTANCE_DECODER] len_state=#{len_state}" if ENV["LZMA_DEBUG_DISTANCE"]
|
|
1533
|
-
XzUtilsDecoderDebug.debug_puts " rep0_before=#{old_rep0}" if old_dict_full.between?(
|
|
1534
|
-
210, 230
|
|
1535
|
-
)
|
|
1536
|
-
puts "[DISTANCE_DECODER] rep0_before=#{old_rep0}" if ENV["LZMA_DEBUG_DISTANCE"]
|
|
1537
|
-
end
|
|
1538
|
-
|
|
1539
|
-
if ENV["LZMA_DEBUG"] && old_dict_full.between?(220, 230)
|
|
1540
|
-
puts "DEBUG decode_regular_match at dict_full=#{old_dict_full}: length=#{length}"
|
|
1541
|
-
end
|
|
1542
|
-
|
|
1543
1000
|
# Decode match distance
|
|
1544
1001
|
# XZ Utils stores distance in rep0 without +1
|
|
1545
1002
|
# The distance coder returns 0-based distance
|
|
1546
1003
|
rep0 = @distance_coder.decode(@range_decoder, len_state)
|
|
1547
1004
|
|
|
1548
|
-
# DEBUG
|
|
1549
|
-
if (ENV.fetch("LZMA_DEBUG",
|
|
1550
|
-
nil) && old_dict_full.between?(210,
|
|
1551
|
-
230)) || old_dict_full == 293
|
|
1552
|
-
puts " rep0_decoded=#{rep0} (distance = #{rep0})"
|
|
1553
|
-
puts " buffer_back calculation: back=#{@dict_full - rep0 - 1}"
|
|
1554
|
-
end
|
|
1555
|
-
if ENV["LZMA_DEBUG"] && rep0 > 100000
|
|
1556
|
-
puts " [LARGE_DISTANCE at dict_full=#{old_dict_full}] rep0=#{rep0}, call count=#{$distance_decode_count || 'unknown'}"
|
|
1557
|
-
end
|
|
1558
|
-
|
|
1559
1005
|
# Check for SDK EOS marker FIRST (before validation)
|
|
1560
1006
|
# EOS marker: rep0 == UINT32_MAX (0xFFFFFFFF)
|
|
1561
1007
|
# XZ Utils checks: if (rep0 == UINT32_MAX) goto eopm;
|
|
@@ -1582,22 +1028,13 @@ check_rc_finished: true)
|
|
|
1582
1028
|
end
|
|
1583
1029
|
end
|
|
1584
1030
|
|
|
1585
|
-
# Validate distance
|
|
1586
|
-
#
|
|
1587
|
-
#
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
unless @dict_full > rep0
|
|
1031
|
+
# Validate distance: ensure we have enough bytes in the buffer.
|
|
1032
|
+
# In this linear buffer model, @pos grows unbounded across chunks while
|
|
1033
|
+
# @dict_full is clamped at @dict_size. Use actual bytes written for validation.
|
|
1034
|
+
actual_bytes_written = @pos - LZ_DICT_INIT_POS
|
|
1035
|
+
unless actual_bytes_written > rep0
|
|
1591
1036
|
raise Omnizip::DecompressionError,
|
|
1592
|
-
"Invalid distance: #{rep0} (
|
|
1593
|
-
end
|
|
1594
|
-
|
|
1595
|
-
# Additional validation: ensure distance doesn't exceed absolute dictionary size
|
|
1596
|
-
# Only validate against absolute dictionary size to prevent buffer overflow
|
|
1597
|
-
# The wrapping logic (below) handles rep0 >= @dict_full cases
|
|
1598
|
-
if rep0 > @dict_size + @dict_full
|
|
1599
|
-
raise Omnizip::DecompressionError,
|
|
1600
|
-
"Invalid distance: #{rep0} exceeds maximum possible (dict_size=#{@dict_size}, dict_full=#{@dict_full})"
|
|
1037
|
+
"Invalid distance: #{rep0} (bytes_written: #{actual_bytes_written})"
|
|
1601
1038
|
end
|
|
1602
1039
|
|
|
1603
1040
|
# IMPORTANT: Limit match length to not exceed uncompressed_size
|
|
@@ -1609,9 +1046,6 @@ check_rc_finished: true)
|
|
|
1609
1046
|
# @uncompressed_size is the target for THIS chunk (not cumulative)
|
|
1610
1047
|
remaining = @uncompressed_size - @chunk_bytes_decoded
|
|
1611
1048
|
if length > remaining
|
|
1612
|
-
if ENV["LZMA_DEBUG"] || remaining <= 5
|
|
1613
|
-
puts "DEBUG: Limiting match length from #{length} to #{remaining} (chunk_bytes_decoded=#{@chunk_bytes_decoded}, uncompressed_size=#{@uncompressed_size}, dict_full=#{@dict_full})"
|
|
1614
|
-
end
|
|
1615
1049
|
length = remaining
|
|
1616
1050
|
end
|
|
1617
1051
|
end
|
|
@@ -1627,75 +1061,28 @@ check_rc_finished: true)
|
|
|
1627
1061
|
#
|
|
1628
1062
|
# dict->size in XZ Utils = dict_size + 2 * LZ_DICT_REPEAT_MAX
|
|
1629
1063
|
# Our dict_buf size = @dict_size + LZ_DICT_INIT_POS = @dict_size + 2 * LZ_DICT_REPEAT_MAX
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
# Our dict_buf size = @dict_size + LZ_DICT_INIT_POS = @dict_size + 2 * LZ_DICT_REPEAT_MAX
|
|
1635
|
-
# So: back += (@dict_size + 2 * LZ_DICT_REPEAT_MAX) - LZ_DICT_REPEAT_MAX
|
|
1636
|
-
# = back + @dict_size + LZ_DICT_REPEAT_MAX
|
|
1637
|
-
back += @dict_size + LZ_DICT_REPEAT_MAX
|
|
1638
|
-
end
|
|
1639
|
-
# Convert to buffer position (add LZ_DICT_INIT_POS offset)
|
|
1640
|
-
buffer_back = back + LZ_DICT_INIT_POS
|
|
1641
|
-
|
|
1642
|
-
# DEBUG: Show buffer position for position 217
|
|
1643
|
-
if old_dict_full.between?(210, 230)
|
|
1644
|
-
XzUtilsDecoderDebug.debug_puts " buffer_back=#{buffer_back}, back=#{back}"
|
|
1645
|
-
bytes_at_back = @dict_buf[buffer_back, 3]
|
|
1646
|
-
bytes_hex = if bytes_at_back.is_a?(String)
|
|
1647
|
-
bytes_at_back.bytes.map do |b|
|
|
1648
|
-
"%02x" % b
|
|
1649
|
-
end.join(" ")
|
|
1650
|
-
else
|
|
1651
|
-
bytes_at_back.map do |b|
|
|
1652
|
-
"%02x" % b
|
|
1653
|
-
end.join(" ")
|
|
1654
|
-
end
|
|
1655
|
-
XzUtilsDecoderDebug.debug_puts " First 3 bytes at buffer_back: #{bytes_hex} (#{bytes_at_back.inspect})"
|
|
1656
|
-
end
|
|
1657
|
-
|
|
1658
|
-
if ENV["LZMA_DEBUG"]
|
|
1659
|
-
b0 = @dict_buf[buffer_back]
|
|
1660
|
-
b1 = @dict_buf[buffer_back + 1]
|
|
1661
|
-
b2 = @dict_buf[buffer_back + 2]
|
|
1662
|
-
b0_str = b0 ? "0x#{b0.to_s(16).upcase}" : "nil"
|
|
1663
|
-
b1_str = b1 ? "0x#{b1.to_s(16).upcase}" : "nil"
|
|
1664
|
-
b2_str = b2 ? "0x#{b2.to_s(16).upcase}" : "nil"
|
|
1665
|
-
b0_chr = b0 ? "'#{b0.chr}'" : "nil"
|
|
1666
|
-
b1_chr = b1 ? "'#{b1.chr}'" : "nil"
|
|
1667
|
-
b2_chr = b2 ? "'#{b2.chr}'" : "nil"
|
|
1668
|
-
warn "DEBUG: copy from buffer_back=#{buffer_back} (distance #{rep0}): #{b0_str} (#{b0_chr}) #{b1_str} (#{b1_chr}) #{b2_str} (#{b2_chr})"
|
|
1669
|
-
warn "DEBUG: pos_before=#{@pos} (output #{@pos - LZ_DICT_INIT_POS}), len=#{length}, pos_after=#{@pos + length} (output #{@pos + length - LZ_DICT_INIT_POS})"
|
|
1670
|
-
# Show what the dictionary contains at key positions (simplified)
|
|
1671
|
-
warn "DEBUG: dict_buf size=#{@dict_buf.size}, allocated=#{@dict_size + 608}"
|
|
1672
|
-
end
|
|
1064
|
+
# Linear buffer: use @pos directly for back reference
|
|
1065
|
+
# @pos always points to the next write position, so @pos - rep0 - 1
|
|
1066
|
+
# gives the correct source position for the match copy
|
|
1067
|
+
buffer_back = @pos - rep0 - 1
|
|
1673
1068
|
|
|
1674
1069
|
# Copy bytes from dictionary and extend buffer as needed
|
|
1675
1070
|
# XZ Utils dict_repeat pattern: dict->buf[dict->pos++] = dict->buf[back++]
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
if ENV["LZMA_DEBUG_ARRAY_WRITE"] && @dict_buf.size != (@dict_size + LZ_DICT_INIT_POS)
|
|
1689
|
-
XzUtilsDecoderDebug.debug_puts "DEBUG: Array expanded during copy! write_pos=#{@pos + i}, byte=#{byte}, old_size=#{@dict_buf.size - 1}, new_size=#{@dict_buf.size}, decoder_id=#{@decoder_id}"
|
|
1690
|
-
end
|
|
1691
|
-
end
|
|
1692
|
-
if ENV["LZMA_DEBUG"] && old_dict_full.between?(220, 230)
|
|
1693
|
-
puts " After copy: #{@dict_buf[@pos, length].inspect}"
|
|
1694
|
-
end
|
|
1071
|
+
src_idx = dict_index(buffer_back)
|
|
1072
|
+
dst_idx = @dict_pos
|
|
1073
|
+
buf_end = @buf_end
|
|
1074
|
+
length.times do
|
|
1075
|
+
byte = @dict_buf.getbyte(src_idx)
|
|
1076
|
+
@dict_buf.setbyte(dst_idx, byte)
|
|
1077
|
+
src_idx += 1
|
|
1078
|
+
src_idx = LZ_DICT_INIT_POS if src_idx >= buf_end
|
|
1079
|
+
dst_idx += 1
|
|
1080
|
+
dst_idx = LZ_DICT_INIT_POS if dst_idx >= buf_end
|
|
1081
|
+
end
|
|
1082
|
+
@dict_pos = dst_idx
|
|
1695
1083
|
|
|
1696
1084
|
# Update state and position
|
|
1697
1085
|
@state.update_match
|
|
1698
|
-
warn "DEBUG: After update_match - state=#{@state.value}" if ENV["LZMA_DEBUG"]
|
|
1699
1086
|
@pos += length
|
|
1700
1087
|
|
|
1701
1088
|
# Update dict_full (XZ Utils pattern)
|
|
@@ -1721,45 +1108,11 @@ check_rc_finished: true)
|
|
|
1721
1108
|
# Update rep distances - rotate and set new rep0
|
|
1722
1109
|
# SDK rotation: rep3←rep2, rep2←rep1, rep1←rep0, rep0←rep0
|
|
1723
1110
|
# XZ Utils stores the actual distance in rep0 (no +1)
|
|
1724
|
-
if ENV["LZMA_DEBUG"]
|
|
1725
|
-
warn "DEBUG: Before rotation - rep0/1/2/3=(#{@rep0},#{@rep1},#{@rep2},#{@rep3}), new distance=#{rep0}"
|
|
1726
|
-
end
|
|
1727
|
-
|
|
1728
|
-
# DEBUG: Trace rep rotation for position 224
|
|
1729
|
-
if ENV["LZMA_DEBUG"] && old_dict_full.between?(220, 230)
|
|
1730
|
-
puts "\n=== Rep rotation after match at dict_full=#{old_dict_full} ==="
|
|
1731
|
-
puts " Before: rep0/1/2/3=(#{@rep0},#{@rep1},#{@rep2},#{@rep3})"
|
|
1732
|
-
puts " Setting rep0 to: #{rep0.inspect}"
|
|
1733
|
-
end
|
|
1734
|
-
|
|
1735
1111
|
@rep3 = @rep2
|
|
1736
1112
|
@rep2 = @rep1
|
|
1737
1113
|
@rep1 = @rep0
|
|
1738
1114
|
@rep0 = rep0
|
|
1739
1115
|
|
|
1740
|
-
if ENV["LZMA_DEBUG"]
|
|
1741
|
-
warn "DEBUG: After rotation - rep0/1/2/3=(#{@rep0},#{@rep1},#{@rep2},#{@rep3})"
|
|
1742
|
-
end
|
|
1743
|
-
|
|
1744
|
-
# DEBUG: Show final rep values
|
|
1745
|
-
if ENV["LZMA_DEBUG"] && old_dict_full.between?(220, 230)
|
|
1746
|
-
puts " After: rep0/1/2/3=(#{@rep0},#{@rep1},#{@rep2},#{@rep3})"
|
|
1747
|
-
end
|
|
1748
|
-
|
|
1749
|
-
# DEBUG: Verify rep0 is actually set
|
|
1750
|
-
if old_dict_full.between?(220, 230)
|
|
1751
|
-
actual_rep0 = @rep0
|
|
1752
|
-
XzUtilsDecoderDebug.debug_puts " VERIFICATION: @rep0=#{actual_rep0.inspect}, @rep0.object_id=#{@rep0.object_id}"
|
|
1753
|
-
end
|
|
1754
|
-
|
|
1755
|
-
# DEBUG: Trace range/code state after match at dict_full 56-62
|
|
1756
|
-
if ENV.fetch("LZMA_DEBUG",
|
|
1757
|
-
nil) && old_dict_full >= 56 && old_dict_full <= 62
|
|
1758
|
-
range_after = @range_decoder.instance_variable_get(:@range)
|
|
1759
|
-
code_after = @range_decoder.instance_variable_get(:@code)
|
|
1760
|
-
XzUtilsDecoderDebug.debug_puts " AFTER match (dict_full #{old_dict_full}→#{@dict_full}): range=0x#{range_after.to_s(16).upcase}, code=0x#{code_after.to_s(16).upcase}"
|
|
1761
|
-
end
|
|
1762
|
-
|
|
1763
1116
|
false # Not EOS, continue decoding
|
|
1764
1117
|
end
|
|
1765
1118
|
|
|
@@ -1777,49 +1130,16 @@ check_rc_finished: true)
|
|
|
1777
1130
|
# @param pos_state [Integer] Position state
|
|
1778
1131
|
# @return [Boolean] Always false (rep matches are never EOS)
|
|
1779
1132
|
def decode_rep_match(pos_state)
|
|
1780
|
-
# DEBUG: Trace rep matches around position 217
|
|
1781
|
-
old_dict_full = @dict_full
|
|
1782
|
-
old_rep0 = @rep0
|
|
1783
|
-
|
|
1784
|
-
# DEBUG: Show rep distances at the start
|
|
1785
|
-
if ENV["LZMA_DEBUG"]
|
|
1786
|
-
warn "DEBUG: decode_rep_match START[#{@decoder_id}] - rep0/1/2/3=(#{@rep0},#{@rep1},#{@rep2},#{@rep3})"
|
|
1787
|
-
end
|
|
1788
|
-
|
|
1789
|
-
# DEBUG: Trace rep matches around position 227
|
|
1790
|
-
if old_dict_full.between?(220, 230)
|
|
1791
|
-
XzUtilsDecoderDebug.debug_puts "\n=== decode_rep_match at dict_full=#{old_dict_full} (decoder_id=#{@decoder_id}) ==="
|
|
1792
|
-
XzUtilsDecoderDebug.debug_puts " At START: rep0/1/2/3=(#{@rep0},#{@rep1},#{@rep2},#{@rep3})"
|
|
1793
|
-
XzUtilsDecoderDebug.debug_puts " old_rep0=#{old_rep0} (captured @rep0)"
|
|
1794
|
-
XzUtilsDecoderDebug.debug_puts " @rep0.object_id=#{@rep0.object_id}"
|
|
1795
|
-
end
|
|
1796
|
-
|
|
1797
1133
|
# Decode which rep distance to use
|
|
1798
1134
|
is_rep0 = @range_decoder.decode_bit(@is_rep0_models[@state.value])
|
|
1799
1135
|
|
|
1800
|
-
puts "DEBUG rep match selection at dict_full=#{@dict_full}: is_rep0=#{is_rep0}, rep0/1/2/3 before=(#{@rep0},#{@rep1},#{@rep2},#{@rep3})" if @dict_full.between?(
|
|
1801
|
-
220, 230
|
|
1802
|
-
)
|
|
1803
|
-
puts " state.value=#{@state.value}, pos_state=#{pos_state}, model_index=#{(@state.value * (1 << @pb)) + pos_state}" if @dict_full.between?(
|
|
1804
|
-
220, 230
|
|
1805
|
-
)
|
|
1806
|
-
|
|
1807
|
-
if ENV["LZMA_DEBUG"]
|
|
1808
|
-
warn "DEBUG: decode_rep_match - is_rep0=#{is_rep0}"
|
|
1809
|
-
end
|
|
1810
|
-
|
|
1811
1136
|
if is_rep0.zero?
|
|
1812
1137
|
# Use rep0
|
|
1813
|
-
puts "DEBUG rep match using rep0" if @dict_full.between?(220, 230)
|
|
1814
1138
|
# XZ Utils: is_rep0_long[state][pos_state] where the array size is NUM_STATES * (1 << pb)
|
|
1815
1139
|
is_rep0_long = @range_decoder.decode_bit(
|
|
1816
|
-
@is_rep0_long_models[(@state.value *
|
|
1140
|
+
@is_rep0_long_models[(@state.value * @pb_shift) + pos_state],
|
|
1817
1141
|
)
|
|
1818
1142
|
|
|
1819
|
-
if ENV["LZMA_DEBUG"]
|
|
1820
|
-
warn "DEBUG: decode_rep_match - is_rep0_long=#{is_rep0_long}"
|
|
1821
|
-
end
|
|
1822
|
-
|
|
1823
1143
|
if is_rep0_long.zero?
|
|
1824
1144
|
# Short rep (length=1)
|
|
1825
1145
|
length = 1
|
|
@@ -1832,10 +1152,6 @@ check_rc_finished: true)
|
|
|
1832
1152
|
end
|
|
1833
1153
|
else
|
|
1834
1154
|
# Not rep0, check rep1/rep2/rep3
|
|
1835
|
-
puts "DEBUG rep match NOT using rep0 (is_rep0=#{is_rep0})" if @dict_full.between?(
|
|
1836
|
-
220, 230
|
|
1837
|
-
)
|
|
1838
|
-
|
|
1839
1155
|
is_rep1 = @range_decoder.decode_bit(@is_rep1_models[@state.value])
|
|
1840
1156
|
|
|
1841
1157
|
if is_rep1.zero?
|
|
@@ -1880,27 +1196,12 @@ check_rc_finished: true)
|
|
|
1880
1196
|
# XZ Utils stores distances without +1 offset
|
|
1881
1197
|
distance = @rep0
|
|
1882
1198
|
|
|
1883
|
-
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
XzUtilsDecoderDebug.debug_puts "\n=== decode_rep_match at dict_full=#{old_dict_full} ==="
|
|
1890
|
-
XzUtilsDecoderDebug.debug_puts " old_rep0=#{old_rep0}, new rep0=#{@rep0} (distance=#{distance})"
|
|
1891
|
-
XzUtilsDecoderDebug.debug_puts " pos_state=#{pos_state}"
|
|
1892
|
-
end
|
|
1893
|
-
|
|
1894
|
-
if ENV["LZMA_DEBUG"]
|
|
1895
|
-
warn "DEBUG: decode_rep_match - length=#{length}, distance=#{distance}, dict_full=#{@dict_full}, rep0/1/2/3=(#{@rep0},#{@rep1},#{@rep2},#{@rep3})"
|
|
1896
|
-
end
|
|
1897
|
-
|
|
1898
|
-
# Validate distance using XZ Utils dict_is_distance_valid pattern
|
|
1899
|
-
# See lz_decoder.h:194-198: return dict->full > distance;
|
|
1900
|
-
# Note: distance=0 is valid (means copy from position 0, the first byte)
|
|
1901
|
-
# XZ Utils allows distance=0 as long as dict->full > 0 (at least one byte available)
|
|
1902
|
-
unless @dict_full > distance
|
|
1903
|
-
raise "Invalid rep distance: #{distance} (dict_full: #{@dict_full})"
|
|
1199
|
+
# Validate distance: ensure we have enough bytes in the buffer.
|
|
1200
|
+
# In this linear buffer model, @pos grows unbounded across chunks while
|
|
1201
|
+
# @dict_full is clamped at @dict_size. Use actual bytes written for validation.
|
|
1202
|
+
actual_bytes_written = @pos - LZ_DICT_INIT_POS
|
|
1203
|
+
unless actual_bytes_written > distance
|
|
1204
|
+
raise "Invalid rep distance: #{distance} (bytes_written: #{actual_bytes_written})"
|
|
1904
1205
|
end
|
|
1905
1206
|
|
|
1906
1207
|
# IMPORTANT: Limit match length to not exceed uncompressed_size
|
|
@@ -1912,9 +1213,6 @@ check_rc_finished: true)
|
|
|
1912
1213
|
# @uncompressed_size is the target for THIS chunk (not cumulative)
|
|
1913
1214
|
remaining = @uncompressed_size - @chunk_bytes_decoded
|
|
1914
1215
|
if length > remaining
|
|
1915
|
-
if ENV["LZMA_DEBUG"] || remaining <= 5
|
|
1916
|
-
puts "DEBUG REP: Limiting rep match length from #{length} to #{remaining} (chunk_bytes_decoded=#{@chunk_bytes_decoded}, uncompressed_size=#{@uncompressed_size}, dict_full=#{@dict_full})"
|
|
1917
|
-
end
|
|
1918
1216
|
length = remaining
|
|
1919
1217
|
end
|
|
1920
1218
|
end
|
|
@@ -1928,79 +1226,23 @@ check_rc_finished: true)
|
|
|
1928
1226
|
#
|
|
1929
1227
|
# dict->size in XZ Utils = dict_size + 2 * LZ_DICT_REPEAT_MAX
|
|
1930
1228
|
# Our dict_buf size = @dict_size + LZ_DICT_INIT_POS = @dict_size + 2 * LZ_DICT_REPEAT_MAX
|
|
1931
|
-
|
|
1932
|
-
|
|
1933
|
-
if distance >= @dict_full
|
|
1934
|
-
# Distance wraps to the end of the dictionary buffer
|
|
1935
|
-
# XZ Utils: back += dict->size - LZ_DICT_REPEAT_MAX;
|
|
1936
|
-
# Our dict_buf size = @dict_size + LZ_DICT_INIT_POS = @dict_size + 2 * LZ_DICT_REPEAT_MAX
|
|
1937
|
-
# So: back += (@dict_size + 2 * LZ_DICT_REPEAT_MAX) - LZ_DICT_REPEAT_MAX
|
|
1938
|
-
# = back + @dict_size + LZ_DICT_REPEAT_MAX
|
|
1939
|
-
back += @dict_size + LZ_DICT_REPEAT_MAX
|
|
1940
|
-
end
|
|
1941
|
-
# Convert to buffer position (add LZ_DICT_INIT_POS offset)
|
|
1942
|
-
buffer_back = back + LZ_DICT_INIT_POS
|
|
1943
|
-
|
|
1944
|
-
puts "DEBUG rep match copy at dict_full=#{@dict_full}: @dict_full=#{@dict_full}, distance=#{distance}, back=#{back}, buffer_back=#{buffer_back}" if @dict_full.between?(
|
|
1945
|
-
220, 230
|
|
1946
|
-
)
|
|
1947
|
-
|
|
1948
|
-
# DEBUG: Show back calculation for position 217
|
|
1949
|
-
if old_dict_full.between?(210, 230)
|
|
1950
|
-
XzUtilsDecoderDebug.debug_puts " back calculation: @dict_full=#{@dict_full}, distance=#{distance}"
|
|
1951
|
-
XzUtilsDecoderDebug.debug_puts " back=#{old_back}, wrapped_back=#{back}, buffer_back=#{buffer_back}"
|
|
1952
|
-
bytes_at_back = @dict_buf[buffer_back, 3]
|
|
1953
|
-
bytes_hex = if bytes_at_back.is_a?(String)
|
|
1954
|
-
bytes_at_back.bytes.map do |b|
|
|
1955
|
-
"%02x" % b
|
|
1956
|
-
end.join(" ")
|
|
1957
|
-
else
|
|
1958
|
-
[bytes_at_back].flatten.map do |b|
|
|
1959
|
-
"%02x" % b
|
|
1960
|
-
end.join(" ")
|
|
1961
|
-
end
|
|
1962
|
-
XzUtilsDecoderDebug.debug_puts " First 3 bytes at buffer_back: #{bytes_hex} (#{bytes_at_back.inspect})"
|
|
1963
|
-
end
|
|
1229
|
+
# Linear buffer: use @pos directly for back reference
|
|
1230
|
+
buffer_back = @pos - distance - 1
|
|
1964
1231
|
|
|
1965
|
-
# Copy bytes from dictionary
|
|
1232
|
+
# Copy bytes from dictionary
|
|
1966
1233
|
# XZ Utils dict_repeat pattern: dict->buf[dict->pos++] = dict->buf[back++]
|
|
1967
|
-
|
|
1968
|
-
|
|
1969
|
-
|
|
1970
|
-
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
1975
|
-
|
|
1976
|
-
|
|
1977
|
-
end
|
|
1978
|
-
|
|
1979
|
-
byte = @dict_buf[buffer_back + i]
|
|
1980
|
-
if @dict_full == 227 && i.zero?
|
|
1981
|
-
puts "DEBUG dict_copy at dict_full=227, i=0:"
|
|
1982
|
-
puts " buffer_back=#{buffer_back}, byte=#{byte} ('#{byte.chr}')"
|
|
1983
|
-
puts " Writing to @pos=#{@pos + i}"
|
|
1984
|
-
puts " dict_buf[buffer_back...buffer_back+10] = #{@dict_buf[buffer_back,
|
|
1985
|
-
10].inspect}"
|
|
1986
|
-
# DEBUG: Check if buffer_back+1 has the correct byte
|
|
1987
|
-
puts " dict_buf[buffer_back+1=#{buffer_back + 1}] = #{@dict_buf[buffer_back + 1].inspect} ('#{begin
|
|
1988
|
-
@dict_buf[buffer_back + 1].chr
|
|
1989
|
-
rescue StandardError
|
|
1990
|
-
'?'
|
|
1991
|
-
end}')"
|
|
1992
|
-
prev_5 = if buffer_back > 4
|
|
1993
|
-
@dict_buf[(buffer_back - 5)..(buffer_back - 1)].map do |b|
|
|
1994
|
-
"0x#{b.to_s(16).upcase} (#{b.chr})"
|
|
1995
|
-
end.join(", ")
|
|
1996
|
-
else
|
|
1997
|
-
"N/A"
|
|
1998
|
-
end
|
|
1999
|
-
puts " Previous 5 bytes: [#{prev_5}]"
|
|
2000
|
-
puts " Current dict_full=#{@dict_full}, @pos=#{@pos}"
|
|
2001
|
-
end
|
|
2002
|
-
@dict_buf[@pos + i] = byte
|
|
2003
|
-
end
|
|
1234
|
+
src_idx = dict_index(buffer_back)
|
|
1235
|
+
dst_idx = @dict_pos
|
|
1236
|
+
buf_end = @buf_end
|
|
1237
|
+
length.times do
|
|
1238
|
+
byte = @dict_buf.getbyte(src_idx)
|
|
1239
|
+
@dict_buf.setbyte(dst_idx, byte)
|
|
1240
|
+
src_idx += 1
|
|
1241
|
+
src_idx = LZ_DICT_INIT_POS if src_idx >= buf_end
|
|
1242
|
+
dst_idx += 1
|
|
1243
|
+
dst_idx = LZ_DICT_INIT_POS if dst_idx >= buf_end
|
|
1244
|
+
end
|
|
1245
|
+
@dict_pos = dst_idx
|
|
2004
1246
|
|
|
2005
1247
|
# Update position
|
|
2006
1248
|
@pos += length
|
|
@@ -2050,40 +1292,19 @@ check_rc_finished: true)
|
|
|
2050
1292
|
def calculate_literal_state
|
|
2051
1293
|
# XZ Utils dict_get0 pattern: dict->buf[dict->pos - 1]
|
|
2052
1294
|
# For array buffer, use @pos - 1 as index
|
|
2053
|
-
|
|
2054
|
-
|
|
2055
|
-
|
|
2056
|
-
|
|
2057
|
-
|
|
2058
|
-
|
|
2059
|
-
|
|
2060
|
-
|
|
2061
|
-
|
|
2062
|
-
|
|
2063
|
-
#
|
|
2064
|
-
|
|
2065
|
-
|
|
2066
|
-
if ENV["LZMA_DEBUG_NIL_BYTE"]
|
|
2067
|
-
raise "DEBUG: prev_byte is nil! decoder_id=#{@decoder_id}, @pos=#{@pos}, @dict_full=#{@dict_full}, @dict_buf.size=#{@dict_buf&.size || 'nil'}, accessing index #{@pos - 1}, nil_count=#{@dict_buf&.count(nil) || 'N/A'}, @dict_buf.object_id=#{@dict_buf&.object_id || 'nil'}"
|
|
2068
|
-
end
|
|
2069
|
-
|
|
2070
|
-
prev_byte = 0
|
|
2071
|
-
end
|
|
2072
|
-
|
|
2073
|
-
if ENV["LZMA_DEBUG_CALC_STATE"]
|
|
2074
|
-
XzUtilsDecoderDebug.debug_puts "DEBUG calc_state[#{@decoder_id}]: pos=#{@pos}, dict_full=#{@dict_full}, @dict_buf.object_id=#{@dict_buf.object_id}, prev_byte=#{prev_byte}"
|
|
2075
|
-
end
|
|
2076
|
-
|
|
2077
|
-
# XZ Utils formula from lzma_common.h:literal_mask_calc
|
|
2078
|
-
# literal_mask = (UINT32_C(0x100) << (lp)) - (UINT32_C(0x100) >> (lc))
|
|
2079
|
-
# For lc=3, lp=0: (256 << 0) - (256 >> 3) = 256 - 32 = 224 (0xE0)
|
|
2080
|
-
literal_mask = (0x100 << @lp) - (0x100 >> @lc)
|
|
2081
|
-
|
|
2082
|
-
# Combine dict_full (actual decoded position) and prev_byte, then apply mask
|
|
2083
|
-
# IMPORTANT: XZ Utils uses dict.pos (which starts at 0 and increments)
|
|
2084
|
-
# omnizip's @pos starts at LZ_DICT_INIT_POS (576), so we use @dict_full instead
|
|
2085
|
-
# This ensures we match XZ Utils's literal state calculation exactly
|
|
2086
|
-
(((@dict_full << 8) + prev_byte) & literal_mask)
|
|
1295
|
+
prev_idx = @dict_pos == LZ_DICT_INIT_POS ? @buf_end - 1 : @dict_pos - 1
|
|
1296
|
+
prev_byte = @dict_full.positive? ? @dict_buf.getbyte(prev_idx) : 0
|
|
1297
|
+
|
|
1298
|
+
# Safeguard: if prev_byte is nil, use 0
|
|
1299
|
+
prev_byte = 0 if prev_byte.nil?
|
|
1300
|
+
|
|
1301
|
+
# Combine current output position and prev_byte, then apply cached mask.
|
|
1302
|
+
# XZ Utils uses dict.pos (continuous write position) for this calculation.
|
|
1303
|
+
# We use @pos - LZ_DICT_INIT_POS (total bytes decoded) instead of @dict_full,
|
|
1304
|
+
# because @dict_full is clamped at @dict_size after the circular buffer wraps,
|
|
1305
|
+
# which would freeze the position-dependent literal model selection.
|
|
1306
|
+
output_pos = @pos - LZ_DICT_INIT_POS
|
|
1307
|
+
((output_pos << 8) + prev_byte) & @literal_mask
|
|
2087
1308
|
end
|
|
2088
1309
|
end
|
|
2089
1310
|
end
|