omnizip 0.3.8 → 0.3.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (201) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop_todo.yml +51 -50
  3. data/Gemfile +2 -0
  4. data/Rakefile +3 -0
  5. data/TODO.refactor/00-overview.md +67 -0
  6. data/TODO.refactor/01-registry-base-class.md +81 -0
  7. data/TODO.refactor/02-error-hierarchy.md +57 -0
  8. data/TODO.refactor/03-autoload-entry-point.md +69 -0
  9. data/TODO.refactor/04-filter-base-consolidation.md +49 -0
  10. data/TODO.refactor/05-send-private-methods.md +64 -0
  11. data/TODO.refactor/06-instance-variable-access.md +69 -0
  12. data/TODO.refactor/07-respond-to-replacement.md +91 -0
  13. data/TODO.refactor/08-spec-doubles.md +27 -0
  14. data/TODO.refactor/09-cli-shared-module.md +35 -0
  15. data/TODO.refactor/10-format-detector-ocp.md +34 -0
  16. data/TODO.refactor/11-convenience-decoupling.md +38 -0
  17. data/TODO.refactor/12-thread-safety.md +8 -0
  18. data/TODO.refactor/13-lutaml-model-migration.md +82 -0
  19. data/TODO.refactor/14-add-missing-specs.md +25 -0
  20. data/lib/omnizip/algorithm.rb +0 -2
  21. data/lib/omnizip/algorithm_registry.rb +29 -57
  22. data/lib/omnizip/algorithms/lzma/bit_model.rb +1 -1
  23. data/lib/omnizip/algorithms/lzma/dictionary.rb +4 -3
  24. data/lib/omnizip/algorithms/lzma/distance_coder.rb +5 -102
  25. data/lib/omnizip/algorithms/lzma/length_coder.rb +4 -92
  26. data/lib/omnizip/algorithms/lzma/literal_decoder.rb +4 -119
  27. data/lib/omnizip/algorithms/lzma/lzip_decoder.rb +2 -0
  28. data/lib/omnizip/algorithms/lzma/match_finder.rb +59 -29
  29. data/lib/omnizip/algorithms/lzma/optimal_encoder.rb +20 -12
  30. data/lib/omnizip/algorithms/lzma/range_decoder.rb +77 -269
  31. data/lib/omnizip/algorithms/lzma/range_encoder.rb +0 -9
  32. data/lib/omnizip/algorithms/lzma/xz_range_encoder_exact.rb +0 -15
  33. data/lib/omnizip/algorithms/lzma/xz_state.rb +0 -8
  34. data/lib/omnizip/algorithms/lzma/xz_utils_decoder.rb +189 -968
  35. data/lib/omnizip/algorithms/lzma2/encoder.rb +0 -2
  36. data/lib/omnizip/algorithms/lzma2/lzma2_chunk.rb +0 -2
  37. data/lib/omnizip/algorithms/lzma2/properties.rb +0 -2
  38. data/lib/omnizip/algorithms/ppmd7/model.rb +1 -1
  39. data/lib/omnizip/algorithms/ppmd8/context.rb +0 -2
  40. data/lib/omnizip/algorithms/ppmd8/model.rb +1 -1
  41. data/lib/omnizip/algorithms/ppmd8.rb +0 -2
  42. data/lib/omnizip/algorithms/ppmd_base.rb +0 -2
  43. data/lib/omnizip/algorithms/sevenzip_lzma2.rb +0 -5
  44. data/lib/omnizip/algorithms/xz_lzma2.rb +0 -5
  45. data/lib/omnizip/algorithms/zstandard/frame/header.rb +3 -5
  46. data/lib/omnizip/algorithms.rb +1 -0
  47. data/lib/omnizip/archive_handler.rb +75 -0
  48. data/lib/omnizip/archive_handlers/tar_handler.rb +51 -0
  49. data/lib/omnizip/archive_handlers/zip_handler.rb +73 -0
  50. data/lib/omnizip/archive_handlers.rb +11 -0
  51. data/lib/omnizip/checksum_registry.rb +23 -96
  52. data/lib/omnizip/cli/shared.rb +31 -0
  53. data/lib/omnizip/cli.rb +6 -43
  54. data/lib/omnizip/commands/archive_create_command.rb +0 -6
  55. data/lib/omnizip/commands/archive_extract_command.rb +0 -4
  56. data/lib/omnizip/commands/archive_list_command.rb +0 -4
  57. data/lib/omnizip/commands/archive_repair_command.rb +0 -2
  58. data/lib/omnizip/commands/archive_verify_command.rb +0 -2
  59. data/lib/omnizip/commands/compress_command.rb +0 -2
  60. data/lib/omnizip/commands/decompress_command.rb +0 -2
  61. data/lib/omnizip/commands/list_command.rb +0 -2
  62. data/lib/omnizip/commands/metadata_command.rb +0 -5
  63. data/lib/omnizip/commands/parity_create_command.rb +0 -2
  64. data/lib/omnizip/commands/parity_repair_command.rb +0 -2
  65. data/lib/omnizip/commands/parity_verify_command.rb +0 -2
  66. data/lib/omnizip/commands/profile_list_command.rb +0 -2
  67. data/lib/omnizip/commands/profile_show_command.rb +0 -2
  68. data/lib/omnizip/convenience.rb +111 -234
  69. data/lib/omnizip/converter/seven_zip_to_zip_strategy.rb +1 -1
  70. data/lib/omnizip/crypto/aes256/cipher.rb +6 -1
  71. data/lib/omnizip/error.rb +22 -18
  72. data/lib/omnizip/filter.rb +5 -3
  73. data/lib/omnizip/filter_registry.rb +74 -98
  74. data/lib/omnizip/filters/bcj2/decoder.rb +196 -130
  75. data/lib/omnizip/filters/bcj2.rb +0 -2
  76. data/lib/omnizip/filters/bcj_x86.rb +0 -2
  77. data/lib/omnizip/filters/filter_base.rb +4 -70
  78. data/lib/omnizip/filters/registry.rb +0 -2
  79. data/lib/omnizip/format_detector.rb +21 -12
  80. data/lib/omnizip/format_registry.rb +79 -38
  81. data/lib/omnizip/formats/bzip2_file.rb +0 -2
  82. data/lib/omnizip/formats/cpio/entry.rb +0 -2
  83. data/lib/omnizip/formats/cpio/reader.rb +0 -1
  84. data/lib/omnizip/formats/cpio/writer.rb +0 -1
  85. data/lib/omnizip/formats/cpio.rb +0 -1
  86. data/lib/omnizip/formats/format_spec_loader.rb +1 -1
  87. data/lib/omnizip/formats/gzip.rb +0 -2
  88. data/lib/omnizip/formats/iso/directory_record.rb +6 -3
  89. data/lib/omnizip/formats/iso/reader.rb +2 -2
  90. data/lib/omnizip/formats/iso.rb +0 -1
  91. data/lib/omnizip/formats/lzip.rb +5 -25
  92. data/lib/omnizip/formats/lzma_alone.rb +7 -25
  93. data/lib/omnizip/formats/msi/cab_extractor.rb +181 -0
  94. data/lib/omnizip/formats/msi/constants.rb +157 -0
  95. data/lib/omnizip/formats/msi/directory_resolver.rb +188 -0
  96. data/lib/omnizip/formats/msi/entry.rb +128 -0
  97. data/lib/omnizip/formats/msi/reader.rb +301 -0
  98. data/lib/omnizip/formats/msi/string_pool.rb +246 -0
  99. data/lib/omnizip/formats/msi/table_parser.rb +244 -0
  100. data/lib/omnizip/formats/msi.rb +84 -0
  101. data/lib/omnizip/formats/ole/allocation_table.rb +10 -1
  102. data/lib/omnizip/formats/ole/dirent.rb +20 -3
  103. data/lib/omnizip/formats/ole/header.rb +0 -1
  104. data/lib/omnizip/formats/ole/ranges_io.rb +21 -3
  105. data/lib/omnizip/formats/ole/storage.rb +19 -4
  106. data/lib/omnizip/formats/ole.rb +0 -2
  107. data/lib/omnizip/formats/rar/block_parser.rb +0 -1
  108. data/lib/omnizip/formats/rar/compression/ppmd/encoder.rb +1 -1
  109. data/lib/omnizip/formats/rar/decompressor.rb +38 -1
  110. data/lib/omnizip/formats/rar/header.rb +0 -1
  111. data/lib/omnizip/formats/rar/rar5/writer.rb +0 -2
  112. data/lib/omnizip/formats/rar/reader.rb +2 -3
  113. data/lib/omnizip/formats/rar/writer.rb +0 -1
  114. data/lib/omnizip/formats/rar.rb +2 -2
  115. data/lib/omnizip/formats/rpm/entry.rb +0 -1
  116. data/lib/omnizip/formats/rpm/header.rb +3 -8
  117. data/lib/omnizip/formats/rpm/lead.rb +37 -26
  118. data/lib/omnizip/formats/rpm/tag.rb +0 -1
  119. data/lib/omnizip/formats/rpm/writer.rb +0 -1
  120. data/lib/omnizip/formats/seven_zip/bcj2_stream_decompressor.rb +98 -61
  121. data/lib/omnizip/formats/seven_zip/coder_chain.rb +1 -2
  122. data/lib/omnizip/formats/seven_zip/encoded_header.rb +0 -1
  123. data/lib/omnizip/formats/seven_zip/encrypted_header.rb +0 -1
  124. data/lib/omnizip/formats/seven_zip/header.rb +0 -1
  125. data/lib/omnizip/formats/seven_zip/header_encryptor.rb +0 -1
  126. data/lib/omnizip/formats/seven_zip/header_writer.rb +22 -61
  127. data/lib/omnizip/formats/seven_zip/models/file_entry.rb +1 -1
  128. data/lib/omnizip/formats/seven_zip/parser.rb +14 -13
  129. data/lib/omnizip/formats/seven_zip/reader.rb +82 -19
  130. data/lib/omnizip/formats/seven_zip/split_archive_reader.rb +0 -15
  131. data/lib/omnizip/formats/seven_zip/split_archive_writer.rb +0 -1
  132. data/lib/omnizip/formats/seven_zip/writer.rb +22 -68
  133. data/lib/omnizip/formats/seven_zip.rb +63 -5
  134. data/lib/omnizip/formats/tar/entry.rb +0 -2
  135. data/lib/omnizip/formats/tar/header.rb +0 -2
  136. data/lib/omnizip/formats/tar/reader.rb +0 -2
  137. data/lib/omnizip/formats/tar/writer.rb +0 -2
  138. data/lib/omnizip/formats/tar.rb +0 -1
  139. data/lib/omnizip/formats/xar/entry.rb +0 -1
  140. data/lib/omnizip/formats/xar/header.rb +0 -1
  141. data/lib/omnizip/formats/xar/reader.rb +0 -1
  142. data/lib/omnizip/formats/xar/toc.rb +3 -3
  143. data/lib/omnizip/formats/xar/writer.rb +11 -3
  144. data/lib/omnizip/formats/xar.rb +0 -1
  145. data/lib/omnizip/formats/xz/reader.rb +7 -3
  146. data/lib/omnizip/formats/xz.rb +3 -40
  147. data/lib/omnizip/formats/xz_impl/block_decoder.rb +4 -10
  148. data/lib/omnizip/formats/xz_impl/block_encoder.rb +0 -2
  149. data/lib/omnizip/formats/xz_impl/block_header_parser.rb +0 -2
  150. data/lib/omnizip/formats/xz_impl/index_decoder.rb +0 -1
  151. data/lib/omnizip/formats/xz_impl/index_encoder.rb +0 -1
  152. data/lib/omnizip/formats/xz_impl/stream_encoder.rb +1 -1
  153. data/lib/omnizip/formats/xz_impl/stream_footer.rb +0 -1
  154. data/lib/omnizip/formats/xz_impl/stream_footer_parser.rb +0 -1
  155. data/lib/omnizip/formats/xz_impl/stream_header.rb +0 -1
  156. data/lib/omnizip/formats/xz_impl/stream_header_parser.rb +0 -2
  157. data/lib/omnizip/formats/zip/central_directory_header.rb +2 -3
  158. data/lib/omnizip/formats/zip/end_of_central_directory.rb +0 -2
  159. data/lib/omnizip/formats/zip/local_file_header.rb +0 -2
  160. data/lib/omnizip/formats/zip/reader.rb +12 -2
  161. data/lib/omnizip/formats/zip/writer.rb +73 -1
  162. data/lib/omnizip/formats/zip/zip64_end_of_central_directory.rb +0 -2
  163. data/lib/omnizip/formats/zip/zip64_end_of_central_directory_locator.rb +0 -2
  164. data/lib/omnizip/formats/zip/zip64_extra_field.rb +0 -2
  165. data/lib/omnizip/formats/zip.rb +0 -1
  166. data/lib/omnizip/formats.rb +9 -22
  167. data/lib/omnizip/implementations/seven_zip/lzma/range_decoder.rb +2 -2
  168. data/lib/omnizip/implementations/seven_zip/lzma/range_encoder.rb +1 -1
  169. data/lib/omnizip/implementations/seven_zip/lzma2/encoder.rb +0 -6
  170. data/lib/omnizip/implementations/xz_utils/lzma2/decoder.rb +18 -185
  171. data/lib/omnizip/implementations/xz_utils/lzma2/encoder.rb +3 -3
  172. data/lib/omnizip/io/source.rb +136 -0
  173. data/lib/omnizip/io.rb +2 -0
  174. data/lib/omnizip/link_handler.rb +7 -3
  175. data/lib/omnizip/metadata/metadata_editor.rb +1 -1
  176. data/lib/omnizip/models/algorithm_metadata.rb +33 -60
  177. data/lib/omnizip/models/compression_options.rb +40 -53
  178. data/lib/omnizip/models/conversion_options.rb +9 -18
  179. data/lib/omnizip/models/parallel_options.rb +25 -0
  180. data/lib/omnizip/optimization_registry.rb +8 -44
  181. data/lib/omnizip/parallel/job_scheduler.rb +26 -8
  182. data/lib/omnizip/parallel/parallel_compressor.rb +10 -20
  183. data/lib/omnizip/parallel/parallel_extractor.rb +20 -11
  184. data/lib/omnizip/parity/par2_repairer.rb +5 -5
  185. data/lib/omnizip/parity/par2_verifier.rb +27 -22
  186. data/lib/omnizip/parity.rb +4 -4
  187. data/lib/omnizip/password/encryption_registry.rb +11 -50
  188. data/lib/omnizip/password/zip_crypto_strategy.rb +1 -4
  189. data/lib/omnizip/pipe/stream_decompressor.rb +2 -3
  190. data/lib/omnizip/profile/archive_profile.rb +0 -2
  191. data/lib/omnizip/profile/binary_profile.rb +0 -2
  192. data/lib/omnizip/profile/compression_profile.rb +0 -1
  193. data/lib/omnizip/profile/profile_detector.rb +0 -2
  194. data/lib/omnizip/profile/text_profile.rb +0 -2
  195. data/lib/omnizip/profile.rb +0 -2
  196. data/lib/omnizip/registry.rb +112 -0
  197. data/lib/omnizip/version.rb +1 -1
  198. data/lib/omnizip/zip/file.rb +20 -9
  199. data/lib/omnizip.rb +26 -55
  200. data/lib/tasks/msi_test_files.rake +66 -0
  201. metadata +57 -6
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require "stringio"
4
+
3
5
  # Copyright (C) 2025 Ribose Inc.
4
6
  #
5
7
  # Permission is hereby granted, free of charge, to any person obtaining a
@@ -40,13 +42,6 @@ module Omnizip
40
42
  # - XZ format (.xz files)
41
43
  # - LZMA2 compression in XZ format
42
44
  #
43
- module XzUtilsDecoderDebug
44
- # Debug helper to conditionally output debug messages
45
- # Set ENV['LZMA_DEBUG'] = 'true' to enable all debug output
46
- def self.debug_puts(*args)
47
- puts(*args) if ENV["LZMA_DEBUG"]
48
- end
49
- end
50
45
 
51
46
  # XZ Utils implementation of LZMA decoder
52
47
  #
@@ -91,13 +86,58 @@ module Omnizip
91
86
  RangeDecoder = LZMA::RangeDecoder
92
87
  SdkStateMachine = Implementations::SevenZip::LZMA::StateMachine
93
88
 
94
- attr_reader :lc, :lp, :pb, :dict_size, :uncompressed_size
89
+ attr_reader :lc, :lp, :pb, :dict_size, :uncompressed_size,
90
+ :range_decoder
95
91
 
96
92
  # XZ Utils dictionary constants (from lz_decoder.h)
97
93
  # See: /Users/mulgogi/src/external/xz/src/liblzma/lz/lz_decoder.h
98
94
  LZ_DICT_REPEAT_MAX = 288
99
95
  LZ_DICT_INIT_POS = 2 * LZ_DICT_REPEAT_MAX # = 576
100
96
 
97
+ # Map a monotonic position to a circular buffer index.
98
+ # Buffer layout: [LZ_DICT_INIT_POS zero bytes] + [dict_size circular region]
99
+ # @pos grows forever; this maps it to the fixed-size buffer.
100
+ # Matches C++ 7-Zip SDK: dic[dicPos] with dicPos wrapping at dicBufSize.
101
+ def dict_index(pos)
102
+ LZ_DICT_INIT_POS + ((pos - LZ_DICT_INIT_POS) % @dict_size)
103
+ end
104
+
105
+ # Flush decoded output from the circular dictionary buffer.
106
+ # Copies bytes from @flush_pos to @pos into the output stream or accumulator.
107
+ # Handles wrap-around when the data spans the circular buffer boundary.
108
+ def flush_circular_output(output)
109
+ return if @pos == @flush_pos
110
+
111
+ length = @pos - @flush_pos
112
+
113
+ if length > @dict_size
114
+ raise Omnizip::DecompressionError,
115
+ "Internal error: flush lag (#{length}) exceeds dict_size (#{@dict_size})"
116
+ end
117
+
118
+ start_idx = dict_index(@flush_pos)
119
+ buf_end = @buf_end
120
+
121
+ if start_idx + length <= buf_end
122
+ # No wrap
123
+ data = @dict_buf.byteslice(start_idx, length)
124
+ else
125
+ # Wraps around the circular boundary
126
+ first_len = buf_end - start_idx
127
+ data = @dict_buf.byteslice(start_idx, first_len)
128
+ data << @dict_buf.byteslice(LZ_DICT_INIT_POS, length - first_len)
129
+ end
130
+
131
+ if output
132
+ output.write(data.force_encoding(Encoding::BINARY))
133
+ else
134
+ @output_accumulator ||= StringIO.new("".b)
135
+ @output_accumulator.write(data)
136
+ end
137
+
138
+ @flush_pos = @pos
139
+ end
140
+
101
141
  # Initialize the SDK-compatible decoder
102
142
  #
103
143
  # @param input [IO] Input stream of compressed data
@@ -116,11 +156,6 @@ module Omnizip
116
156
  def initialize(input, options = {})
117
157
  @input = input
118
158
  @decoder_id = object_id # Track decoder instance ID
119
- puts "DEBUG LZMA::Decoder.new created[#{@decoder_id}]" if ENV["LZMA_DEBUG_DECODE_STREAM"]
120
- if ENV["LZMA_DEBUG_DECODE_STREAM"]
121
- warn "SDK Decoder #{@decoder_id} created"
122
- end
123
- @input = input
124
159
 
125
160
  # Check for preloaded data (from LZMA2 uncompressed chunks)
126
161
  @preloaded_data = options[:preloaded_data]
@@ -140,9 +175,15 @@ module Omnizip
140
175
  # Standalone LZMA file - read header from input
141
176
  read_header
142
177
  end
178
+
143
179
  validate_parameters
144
180
  init_models
145
181
  init_coders
182
+
183
+ # Cache computed values for hot loop
184
+ @pb_mask = (1 << @pb) - 1
185
+ @pb_shift = 1 << @pb
186
+ @literal_mask = (0x100 << @lp) - (0x100 >> @lc)
146
187
  end
147
188
 
148
189
  # Decode a compressed stream
@@ -167,46 +208,17 @@ module Omnizip
167
208
  check_rc_finished: true)
168
209
  @decode_stream_call_count ||= 0
169
210
  @decode_stream_call_count += 1
170
- call_num = @decode_stream_call_count
171
-
172
- puts "DEBUG decode_stream START (call ##{call_num}): @pos=#{@pos.inspect}, @dict_full=#{@dict_full.inspect}, preserve_dict=#{preserve_dict}, @uncompressed_size=#{@uncompressed_size.inspect}" if ENV.fetch(
173
- "LZMA_DEBUG", nil
174
- ) && @dict_full && @dict_full >= 200 && @dict_full <= 230
175
- if ENV["LZMA_DEBUG_DECODE_STREAM"]
176
- warn "DEBUG decode_stream[#{@decoder_id}] START: preserve_dict=#{preserve_dict}, @pos=#{@pos.inspect}, @dict_full=#{@dict_full.inspect}, @dict_buf.object_id=#{@dict_buf&.object_id || 'nil'}, @dict_buf.size=#{@dict_buf&.size || 'nil'}"
177
- end
178
-
179
- # Initialize range decoder
180
- # For LZMA2, reuse persistent range decoder across chunks (like XZ Utils)
181
- # The range decoder is created in set_input when the first chunk is processed
182
- # See: /Users/mulgogi/src/external/xz/src/liblzma/lzma/lzma2_decoder.c:140-141
183
- if ENV["LZMA_DEBUG"]
184
- warn "DEBUG: decode_stream - reusing range decoder @input.pos=#{begin
185
- @input.pos
186
- rescue StandardError
187
- 'N/A'
188
- end}, @range_decoder.class=#{@range_decoder.class}"
189
- end
190
211
 
191
212
  # Create range decoder if it doesn't exist (first chunk)
192
213
  # This happens when the decoder is created directly for LZMA (not LZMA2)
193
- unless @range_decoder
194
- if ENV["LZMA_DEBUG"]
195
- warn "DEBUG: decode_stream - creating NEW range decoder"
196
- end
197
- @range_decoder = RangeDecoder.new(@input)
198
- end
214
+ @range_decoder ||= RangeDecoder.new(@input)
199
215
 
200
216
  # Special case: empty input (uncompressed_size == 0)
201
217
  # Return immediately without trying to decode anything
202
218
  if @uncompressed_size != 0xFFFFFFFFFFFFFFFF && @uncompressed_size.zero?
203
- if ENV["LZMA_DEBUG"]
204
- warn "DEBUG: decode_stream - empty input (uncompressed_size=0), returning immediately"
205
- end
206
219
  return "" # Empty output
207
220
  end
208
221
 
209
- # TEMP DEBUG: Trace first 10 iterations
210
222
  @debug_iter = 0
211
223
 
212
224
  # Track bytes decoded in this chunk (for multi-chunk streams)
@@ -214,13 +226,6 @@ check_rc_finished: true)
214
226
  # represents only the current chunk's size, not the total size
215
227
  @chunk_bytes_decoded = 0
216
228
 
217
- # DEBUG: Show chunk_bytes_decoded initialization
218
- if @dict_full && @dict_full >= 220 && @dict_full <= 240 && ENV.fetch(
219
- "LZMA_DEBUG", nil
220
- )
221
- puts "DEBUG: chunk_bytes_decoded reset to 0 for chunk (call_num=#{call_num}, dict_full=#{@dict_full})"
222
- end
223
-
224
229
  # Initialize state and dictionary (XZ Utils system from lz_decoder.c)
225
230
  # See: /Users/mulgogi/src/external/xz/src/liblzma/lz/lz_decoder.c:56
226
231
  # For LZMA2 multi-chunk streams, state machine persists across chunks
@@ -235,38 +240,31 @@ check_rc_finished: true)
235
240
  # when preserve_dict is true (control >= 0x80 but < 0xA0)
236
241
  # For subsequent chunks, the reset() method handles dictionary reset
237
242
  # For the first chunk (when @dict_buf is nil), we need to init it here
238
- puts "DEBUG: Checking @dict_buf.nil? = #{@dict_buf.nil?}, preserve_dict=#{preserve_dict}" if ENV["LZMA_DEBUG_RESET"]
239
243
  if @dict_buf.nil?
240
- buf_size = @dict_size + LZ_DICT_INIT_POS
241
- @dict_buf = Array.new(buf_size, 0)
244
+ @buf_end = LZ_DICT_INIT_POS + @dict_size
245
+ buf_size = @buf_end
246
+ @dict_buf = ("\0" * buf_size).b
242
247
  @pos = LZ_DICT_INIT_POS
248
+ @dict_pos = LZ_DICT_INIT_POS # Circular buffer write position (tracks @pos)
243
249
  @dict_full = 0
244
250
  @has_wrapped = false
245
251
 
246
252
  # Add preloaded data to dictionary (from LZMA2 uncompressed chunks)
247
253
  # This must be done before decoding so matches can reference this data
248
254
  if @preloaded_data && !@preloaded_data.empty?
249
- if ENV["LZMA_DEBUG_RESET"]
250
- warn "DEBUG: Preloading #{@preloaded_data.bytesize} bytes into dictionary[#{@decoder_id}]"
251
- end
255
+ idx = @dict_pos
256
+ buf_end = @buf_end
252
257
  @preloaded_data.each_byte do |byte|
253
- @dict_buf[@pos] = byte
258
+ @dict_buf.setbyte(idx, byte)
254
259
  @pos += 1
260
+ idx += 1
261
+ idx = LZ_DICT_INIT_POS if idx >= buf_end
255
262
  end
263
+ @dict_pos = idx
256
264
  # Update dict_full to reflect preloaded data
257
265
  @dict_full = @pos - LZ_DICT_INIT_POS
258
- if ENV["LZMA_DEBUG_RESET"]
259
- warn "DEBUG: After preload - @pos=#{@pos}, @dict_full=#{@dict_full}"
260
- warn " Preloaded data (hex): #{@preloaded_data[0..50].unpack1('H*')}"
261
- end
262
266
  @preloaded_data = nil # Clear after loading
263
267
  end
264
-
265
- if ENV["LZMA_DEBUG_RESET"]
266
- warn "DEBUG: Dictionary init in decode_stream[#{@decoder_id}] - @pos=#{@pos}, @dict_full=#{@dict_full}, @dict_buf.size=#{@dict_buf.size}, @dict_buf.object_id=#{@dict_buf.object_id}"
267
- # Verify buffer initialization by checking a few positions
268
- warn " Sample values: [576]=#{@dict_buf[576]}, [577]=#{@dict_buf[577]}, [578]=#{@dict_buf[578]}, [583]=#{@dict_buf[583]}"
269
- end
270
268
  end
271
269
 
272
270
  # Track starting position for multi-chunk streams
@@ -275,13 +273,6 @@ check_rc_finished: true)
275
273
  # properly reflected in start_pos, so we only return NEW bytes.
276
274
  # For LZMA2, we need to return only the NEW bytes, not all bytes from LZ_DICT_INIT_POS
277
275
  start_pos = @pos || LZ_DICT_INIT_POS
278
- puts "DEBUG: start_pos=#{start_pos}, @pos=#{@pos.inspect}, @dict_full=#{@dict_full.inspect}, preserve_dict=#{preserve_dict}, @decoder_id=#{@decoder_id}" if ENV.fetch(
279
- "LZMA_DEBUG", nil
280
- ) && @dict_full && @dict_full >= 200 && @dict_full <= 230
281
- # Also show for chunk #1 start (dict_full around 227)
282
- puts "DEBUG: start_pos=#{start_pos}, @pos=#{@pos.inspect}, @dict_full=#{@dict_full.inspect}, @uncompressed_size=#{@uncompressed_size}, @decoder_id=#{@decoder_id}" if ENV.fetch(
283
- "LZMA_DEBUG", nil
284
- ) && @dict_full && @dict_full >= 225 && @dict_full <= 230
285
276
 
286
277
  # Initialize rep distances (XZ Utils initializes to 0)
287
278
  # See: /Users/mulgogi/src/external/xz/src/liblzma/lzma/lzma_decoder.c:1054-1055
@@ -290,15 +281,17 @@ check_rc_finished: true)
290
281
  #
291
282
  # IMPORTANT: Initialize rep distances if they're nil OR not preserving dict
292
283
  if @rep0.nil? || @rep1.nil? || @rep2.nil? || @rep3.nil? || !preserve_dict
293
- puts "DEBUG: Resetting rep distances to 0 (rep0.nil?=#{@rep0.nil?}, preserve_dict=#{preserve_dict})" if ENV.fetch(
294
- "LZMA_DEBUG", nil
295
- ) && @dict_full && @dict_full >= 200 && @dict_full <= 230
296
284
  @rep0 = 0
297
285
  @rep1 = 0
298
286
  @rep2 = 0
299
287
  @rep3 = 0
300
288
  end
301
289
 
290
+ # Read range decoder init bytes (must happen after set_input sets correct stream)
291
+ if @range_decoder.respond_to?(:read_init_bytes) && @range_decoder.init_bytes_remaining.positive?
292
+ @range_decoder.read_init_bytes
293
+ end
294
+
302
295
  # Main decoding loop
303
296
  # XZ Utils pattern (lzma_decoder.c:305-306):
304
297
  # Set dict.limit = dict.pos + (size_t)(coder->uncompressed_size)
@@ -313,178 +306,42 @@ check_rc_finished: true)
313
306
  start_pos + @uncompressed_size
314
307
  end
315
308
 
316
- # DEBUG: Show limit calculation for chunk #1
317
- if ENV.fetch("LZMA_DEBUG_LIMIT",
318
- nil) && @dict_full && @dict_full >= 220 && @dict_full <= 240
319
- puts "DEBUG LIMIT CALCULATION: start_pos=#{start_pos}, @uncompressed_size=#{@uncompressed_size}, limit=#{limit.inspect}"
320
- end
321
- # DEBUG: Also show for dict_full around 293 (where the error occurs)
322
- if ENV.fetch("LZMA_DEBUG_LIMIT",
323
- nil) && @dict_full && @dict_full >= 290 && @dict_full <= 300
324
- puts "DEBUG LIMIT CALCULATION at dict_full=#{@dict_full}: start_pos=#{start_pos}, @uncompressed_size=#{@uncompressed_size}, limit=#{limit.inspect}, @decoder_id=#{@decoder_id}"
325
- end
309
+ # Circular buffer: no growth needed. Buffer is fixed at dict_size + LZ_DICT_INIT_POS.
310
+ # Output is flushed incrementally when the circular position nears wrap point.
311
+ # Matches C++ 7-Zip SDK: dic is pre-allocated to dicBufSize and never resized.
312
+ @flush_pos = @pos
313
+ @output_accumulator = output.nil? ? StringIO.new("".b) : nil
326
314
 
327
315
  iteration = 0
328
316
  loop do
329
317
  iteration += 1
330
- # DEBUG: Show every iteration after position 200
331
- if ENV.fetch("LZMA_DEBUG_ITER",
332
- nil) && @dict_full && @dict_full >= 200 && @dict_full <= 500
333
- puts "DEBUG ITERATION ##{iteration}: pos=#{@pos}, dict_full=#{@dict_full}, limit=#{limit.inspect}"
334
- end
335
318
  # Check if we've reached the expected size (if known)
336
319
  # XZ Utils: checks dict.pos < dict.limit
337
- if ENV["LZMA_DEBUG_LIMIT"]
338
- compare_result = begin
339
- limit && @pos >= limit
340
- rescue StandardError
341
- "ERROR"
342
- end
343
- XzUtilsDecoderDebug.debug_puts "DEBUG LIMIT: iter=#{iteration}, pos=#{@pos.inspect}, dict_full=#{@dict_full}, limit=#{limit.inspect}, pos >= limit: #{compare_result}"
344
- end
345
320
 
346
321
  # Handle nil @pos or limit gracefully
347
322
  if limit && (@pos.nil? || limit.nil?)
348
323
  raise "Invalid state: @pos=#{@pos.inspect}, limit=#{limit.inspect}"
349
324
  end
350
325
 
351
- if ENV["LZMA_DEBUG_LIMIT"]
352
- XzUtilsDecoderDebug.debug_puts "DEBUG LIMIT: iter=#{iteration}, pos=#{@pos}, dict_full=#{@dict_full}, limit=#{limit}"
326
+ # Circular buffer: flush output before it can be overwritten.
327
+ # LZMA matches are at most 273 bytes, so flush when within 273 of wrapping.
328
+ if @pos - @flush_pos > @dict_size - 273
329
+ flush_circular_output(output)
353
330
  end
354
331
 
355
- # DEBUG: Track position before decoding
356
- @pos if ENV["LZMA_DEBUG_POS"]
357
-
358
332
  # Decode is_match bit
359
- pos_state = @pos & ((1 << @pb) - 1)
333
+ pos_state = @pos & @pb_mask
360
334
  # XZ Utils: is_match[state][pos_state] where the array is NUM_STATES * (1 << pb)
361
335
  # The array stride changes with pb value
362
- model_index = (@state.value * (1 << @pb)) + pos_state
336
+ model_index = (@state.value * @pb_shift) + pos_state
363
337
 
364
- # DEBUG: Show state before decode (for position tracking)
365
- if ENV["LZMA_DEBUG_POS_227"]
366
- XzUtilsDecoderDebug.debug_puts "DEBUG: Before is_match at pos=#{@pos}, dict_full=#{@dict_full}, state=#{@state.value}, pos_state=#{pos_state}"
367
- end
368
-
369
- # Debug trace (disabled - remove or enable with ENV var as needed)
370
338
  @debug_iter += 1
371
339
 
372
- # DEBUG: Trace is_match decision around position 256
373
- # IMPORTANT: Capture range/code BEFORE calling decode_bit
374
- if @dict_full.between?(255, 257)
375
- range = @range_decoder.instance_variable_get(:@range)
376
- code = @range_decoder.instance_variable_get(:@code)
377
- model = @is_match_models[model_index]
378
- XzUtilsDecoderDebug.debug_puts " [IS_MATCH] pos=#{@pos}, dict_full=#{@dict_full}, state=#{@state.value}, pos_state=#{pos_state}, model_index=#{model_index}"
379
- XzUtilsDecoderDebug.debug_puts " BEFORE decode: range=0x#{range.to_s(16)}, code=0x#{code.to_s(16)}, prob=#{model.probability}"
380
- # Enable detailed tracing for the critical is_match[0][0] decision
381
- $trace_is_match_0_0 = (@dict_full == 256)
382
- end
383
-
384
- if ENV.fetch("LZMA_DEBUG_ITER", nil)
385
- range = @range_decoder.instance_variable_get(:@range)
386
- code = @range_decoder.instance_variable_get(:@code)
387
- model = @is_match_models[model_index]
388
- bound = (range >> 11) * model.probability
389
- XzUtilsDecoderDebug.debug_puts ""
390
- XzUtilsDecoderDebug.debug_puts "ITER #{@debug_iter}:"
391
- XzUtilsDecoderDebug.debug_puts " pos=#{@pos}, state=#{@state.value}, pos_state=#{pos_state}, model_index=#{model_index}"
392
- XzUtilsDecoderDebug.debug_puts " dict_full=#{@dict_full}"
393
- XzUtilsDecoderDebug.debug_puts " range=0x#{range.to_s(16)}, code=0x#{code.to_s(16)}, model.prob=#{model.probability}"
394
- XzUtilsDecoderDebug.debug_puts " bound=0x#{bound.to_s(16)}, code < bound: #{code < bound}"
395
- end
396
-
397
340
  is_match = @range_decoder.decode_bit(@is_match_models[model_index])
398
341
 
399
- # DEBUG: Trace is_match and literal/match decisions around dict_full = 50-62
400
- if @dict_full.between?(50, 62)
401
- range_val = @range_decoder.instance_variable_get(:@range)
402
- code_val = @range_decoder.instance_variable_get(:@code)
403
- prob_val = @is_match_models[model_index].probability
404
- XzUtilsDecoderDebug.debug_puts "\n=== dict_full=#{@dict_full}: is_match=#{is_match}, state=#{@state.value}, pos_state=#{pos_state} ==="
405
- XzUtilsDecoderDebug.debug_puts " model_index=#{model_index}, prob=#{prob_val}"
406
- XzUtilsDecoderDebug.debug_puts " range=0x#{range_val.to_s(16).upcase}, code=0x#{code_val.to_s(16).upcase}"
407
- end
408
-
409
- if ENV.fetch("LZMA_DEBUG_ITER", nil)
410
- XzUtilsDecoderDebug.debug_puts " is_match=#{is_match}"
411
- end
412
-
413
- # DEBUG: Show is_match result after decode
414
- if @dict_full.between?(255, 257)
415
- XzUtilsDecoderDebug.debug_puts " AFTER decode: is_match=#{is_match}"
416
- XzUtilsDecoderDebug.debug_puts " (is_match=0 means literal, is_match=1 means match)"
417
- end
418
-
419
- # DEBUG: Track what's happening around dict_full=227 (corruption point)
420
- if ENV["LZMA_DEBUG"] && @dict_full == 227
421
- puts "DEBUG CORRUPTION POINT: dict_full=#{@dict_full}, pos=#{@pos}"
422
- puts " is_match=#{is_match}, state=#{@state.value}"
423
- range_val = @range_decoder.instance_variable_get(:@range)
424
- code_val = @range_decoder.instance_variable_get(:@code)
425
- puts " range=0x#{range_val.to_s(16)}, code=0x#{code_val.to_s(16)}"
426
- puts " dict_buf[#{@pos - 5}...#{@pos + 5}] = #{@dict_buf[[
427
- @pos - 5, LZ_DICT_INIT_POS
428
- ].max...[@pos + 5, @dict_buf.size - 1].min].inspect}"
429
- end
430
-
431
- if ENV["LZMA_DEBUG"] && @dict_full.between?(224, 235)
432
- puts "DEBUG pos #{@dict_full}: is_match=#{is_match}, state=#{@state.value}"
433
- if is_match.zero?
434
- puts " Next byte should be literal"
435
- else
436
- puts " Next byte should be match"
437
- end
438
- end
439
-
440
- # DEBUG: Verify first 256 bytes are correct
441
- if @dict_full == 256
442
- XzUtilsDecoderDebug.debug_puts ""
443
- XzUtilsDecoderDebug.debug_puts " Verifying first 256 bytes:"
444
- # Check specific bytes around position 253
445
- XzUtilsDecoderDebug.debug_puts " Byte 253: @dict_buf[#{LZ_DICT_INIT_POS + 253}]=#{@dict_buf[LZ_DICT_INIT_POS + 253].inspect} (expected 'i'=0x69)"
446
- XzUtilsDecoderDebug.debug_puts " Byte 254: @dict_buf[#{LZ_DICT_INIT_POS + 254}]=#{@dict_buf[LZ_DICT_INIT_POS + 254].inspect} (expected 'n'=0x6E)"
447
- XzUtilsDecoderDebug.debug_puts " Byte 255: @dict_buf[#{LZ_DICT_INIT_POS + 255}]=#{@dict_buf[LZ_DICT_INIT_POS + 255].inspect} (expected ' '=0x20)"
448
- all_correct = true
449
- 256.times do |i|
450
- expected = i
451
- actual = @dict_buf[LZ_DICT_INIT_POS + i]
452
- if actual != expected
453
- all_correct = false
454
- if (i >= 253) && ENV.fetch("LZMA_DEBUG", nil)
455
- puts " Byte #{i}: expected 0x#{expected.to_s(16)}, got 0x#{actual.to_s(16)} MISMATCH!"
456
- end
457
- end
458
- end
459
- XzUtilsDecoderDebug.debug_puts " First 256 bytes: #{all_correct ? 'ALL CORRECT ✓' : 'HAS MISMATCH'}"
460
- XzUtilsDecoderDebug.debug_puts ""
461
- end
462
-
463
- if ENV.fetch("LZMA_DEBUG", nil) && @pos >= 605 && @pos <= 615
464
- warn "DEBUG: is_match at pos=#{@pos}, state=#{@state.value}, pos_state=#{pos_state}, model_index=#{model_index}, is_match=#{is_match}"
465
- end
466
-
467
342
  if is_match.zero?
468
343
  # Decode literal
469
344
  decode_literal
470
-
471
- # Trace positions 45-65 for debugging good-1-lzma2-3.xz divergence
472
- if ENV.fetch("LZMA_DEBUG",
473
- nil) && @dict_full >= 45 && @dict_full <= 65
474
- last_byte = @dict_buf[@pos - 1]
475
- range_after = @range_decoder.instance_variable_get(:@range)
476
- code_after = @range_decoder.instance_variable_get(:@code)
477
- puts " literal decoded: 0x#{last_byte.to_s(16).upcase} ('#{last_byte.chr}') at pos=#{@pos - 1}, dict_full=#{@dict_full}"
478
- puts " AFTER: range=0x#{range_after.to_s(16).upcase}, code=0x#{code_after.to_s(16).upcase}"
479
- end
480
-
481
- if ENV.fetch("LZMA_DEBUG_ITER", nil)
482
- last_byte = @dict_buf[@pos - 1]
483
- puts " literal byte=0x#{last_byte.to_s(16)} ('#{last_byte.chr}')"
484
- end
485
- if ENV["LZMA_DEBUG_POS"] && @pos >= limit
486
- puts "DEBUG: Literal overshoot: pos=#{@pos}, limit=#{limit}, delta=#{@pos - limit}"
487
- end
488
345
  elsif decode_match
489
346
  # Decode match - returns true if EOS detected
490
347
  break
@@ -496,7 +353,6 @@ check_rc_finished: true)
496
353
  # IMPORTANT: Must verify range decoder is finished (code == 0)
497
354
  # If code != 0, there's leftover data in the compressed stream (corruption)
498
355
  if limit && @pos >= limit
499
- puts "DEBUG LIMIT TRIGGERED (call #{call_num}): pos=#{@pos}, limit=#{limit}, dict_full=#{@dict_full}, chunk_bytes_decoded=#{@chunk_bytes_decoded}" if ENV["LZMA_DEBUG_LIMIT"]
500
356
 
501
357
  # XZ Utils pattern (lzma_decoder.c:689-700):
502
358
  # Check if range decoder is finished (code == 0)
@@ -531,32 +387,23 @@ check_rc_finished: true)
531
387
  raise Omnizip::DecompressionError,
532
388
  "LZMA stream finished with leftover compressed data (range_decoder.code=#{@range_decoder.code}, expected 0). This indicates corruption in the compressed stream or an invalid EOPM for LZMA2."
533
389
  end
390
+ # XZ Utils pattern (lzma_decoder.c): when STREAM_END is reached,
391
+ # reset the range coder for the next chunk. This happens even for
392
+ # no-reset chunks (control 0x80-0x9F) - the range coder is ALWAYS
393
+ # re-initialized between chunks, only state/dict/models are preserved.
394
+ @range_decoder.reset
534
395
  break
535
396
  elsif @range_decoder.code.zero?
536
397
  # EOPM is allowed (e.g., LZMA_Alone format)
537
398
  # If range decoder is finished, we're done
399
+ # XZ Utils: rc_reset at STREAM_END
400
+ @range_decoder.reset
538
401
  break
539
402
  # Otherwise, continue decoding to find EOPM marker
540
403
  # XZ Utils sets eopm_is_valid = true and continues
541
404
  # Reference: lzma_decoder.c:704
542
405
  end
543
406
  end
544
-
545
- # DEBUG: Show when approaching limit for chunk #1
546
- if ENV.fetch("LZMA_DEBUG_LIMIT",
547
- nil) && limit && @pos >= limit - 10 && @pos < limit + 10
548
- puts "DEBUG NEAR LIMIT (call #{call_num}): pos=#{@pos}, limit=#{limit}, dict_full=#{@dict_full}, chunk_bytes_decoded=#{@chunk_bytes_decoded}, remaining=#{@uncompressed_size ? @uncompressed_size - @chunk_bytes_decoded : 'N/A'}"
549
- end
550
-
551
- # DEBUG: Show when we've passed the expected limit
552
- if ENV.fetch("LZMA_DEBUG_LIMIT",
553
- nil) && limit && @pos >= limit && @pos < limit + 10
554
- puts "DEBUG PASSED LIMIT: pos=#{@pos}, limit=#{limit}, dict_full=#{@dict_full}, delta=#{@pos - limit}"
555
- end
556
-
557
- if ENV["LZMA_DEBUG_POS"] && @pos >= limit
558
- XzUtilsDecoderDebug.debug_puts "DEBUG: Overshoot detected: pos=#{@pos}, limit=#{limit}, delta=#{@pos - limit}"
559
- end
560
407
  end
561
408
 
562
409
  # Validate decoded size against expected uncompressed_size
@@ -585,8 +432,8 @@ check_rc_finished: true)
585
432
  #
586
433
  # If EOPM was NOT encountered (code != 0), leftover data is expected
587
434
  # (it's part of the compressed stream that we haven't read yet).
588
- if @allow_eopm && @range_decoder&.code&.zero? && @range_decoder.instance_variable_get(:@stream)
589
- stream = @range_decoder.instance_variable_get(:@stream)
435
+ if @allow_eopm && @range_decoder&.code&.zero? && @range_decoder&.stream
436
+ stream = @range_decoder.stream
590
437
  # Try to peek at the next byte - if available, there's data AFTER EOPM
591
438
  begin
592
439
  next_byte = stream.getbyte
@@ -599,9 +446,9 @@ check_rc_finished: true)
599
446
  rescue IOError, EOFError
600
447
  # Stream doesn't support peeking or is exhausted, that's fine
601
448
  end
602
- elsif !@allow_eopm && @range_decoder&.instance_variable_get(:@stream)
449
+ elsif !@allow_eopm && @range_decoder&.stream
603
450
  # For LZMA2 mode (EOPM not allowed): check for leftover data
604
- stream = @range_decoder.instance_variable_get(:@stream)
451
+ stream = @range_decoder.stream
605
452
  begin
606
453
  next_byte = stream.getbyte
607
454
  if next_byte
@@ -615,34 +462,14 @@ check_rc_finished: true)
615
462
  end
616
463
  end
617
464
 
618
- # Return output - only the valid portion of dictionary
619
- # XZ Utils: valid data starts from LZ_DICT_INIT_POS onwards
620
- # IMPORTANT: For LZMA2 multi-chunk streams, only return NEW bytes since start_pos!
621
- # This ensures each chunk returns only its own output, not previous chunks' output.
622
- if ENV["DEBUG_DICT_BUF"]
623
- XzUtilsDecoderDebug.debug_puts "DEBUG: start_pos=#{start_pos}, @pos=#{@pos.inspect}, @dict_buf.size=#{@dict_buf.size}, LZ_DICT_INIT_POS=#{LZ_DICT_INIT_POS}"
624
- end
625
- valid_bytes = @dict_buf[start_pos...@pos]
626
- # DEBUG: Show return value calculation
627
- puts "DEBUG RETURN CALCULATION: call_num=#{call_num}, start_pos=#{start_pos}, @pos=#{@pos}, valid_bytes.size=#{@dict_buf[start_pos...@pos].size}, dict_full=#{@dict_full}, chunk_bytes_decoded=#{@chunk_bytes_decoded}" if ENV.fetch(
628
- "LZMA_DEBUG", nil
629
- ) && @dict_full && @dict_full >= 220 && @dict_full <= 240 && call_num == 2
630
- puts "DEBUG RETURN CALCULATION: call_num=#{call_num}, start_pos=#{start_pos}, @pos=#{@pos}, valid_bytes.size=#{@dict_buf[start_pos...@pos].size}, dict_full=#{@dict_full}, chunk_bytes_decoded=#{@chunk_bytes_decoded}" if ENV["LZMA_DEBUG"] && call_num == 2
631
- # Filter out nil values (can happen during dictionary reset transitions)
632
- valid_bytes = valid_bytes.map { |b| b.nil? ? 0 : b }
633
- if ENV["DEBUG_DICT_BUF"]
634
- XzUtilsDecoderDebug.debug_puts "DEBUG: valid_bytes=#{begin
635
- valid_bytes.size
636
- rescue StandardError
637
- valid_bytes.inspect
638
- end}"
639
- end
640
- valid_data = valid_bytes.pack("C*")
465
+ # Flush remaining output from the circular buffer
466
+ flush_circular_output(output)
467
+
468
+ # Return output
641
469
  if output
642
- output.write(valid_data.force_encoding(Encoding::BINARY))
643
- valid_data.bytesize
470
+ @pos - start_pos
644
471
  else
645
- valid_data.force_encoding(Encoding::BINARY)
472
+ @output_accumulator.string.force_encoding(Encoding::BINARY)
646
473
  end
647
474
  end
648
475
 
@@ -660,23 +487,19 @@ check_rc_finished: true)
660
487
  # @param preserve_dict [Boolean] If true, preserve dictionary state (pos, dict_full)
661
488
  # @return [void]
662
489
  def reset(new_lc: nil, new_lp: nil, new_pb: nil, preserve_dict: false)
663
- if ENV["LZMA_DEBUG_RESET"]
664
- warn "DEBUG reset[#{@decoder_id}] called: preserve_dict=#{preserve_dict}, @pos=#{@pos.inspect}, @dict_full=#{@dict_full.inspect}, @dict_buf.size=#{@dict_buf&.size || 'nil'}, nil_count=#{@dict_buf&.count(nil) || 'N/A'}"
665
- end
666
-
667
- # DEBUG: Trace reset calls around position 224-227
668
- if @dict_full && @dict_full >= 220 && @dict_full <= 230
669
- XzUtilsDecoderDebug.debug_puts "\n=== reset called at dict_full=#{@dict_full} ==="
670
- XzUtilsDecoderDebug.debug_puts " preserve_dict=#{preserve_dict}"
671
- XzUtilsDecoderDebug.debug_puts " Before reset: rep0/1/2/3=(#{@rep0},#{@rep1},#{@rep2},#{@rep3})"
672
- end
673
-
674
490
  # Update properties if provided
675
491
  properties_changed = !!(new_lc || new_lp || new_pb)
676
492
  @lc = new_lc if new_lc
677
493
  @lp = new_lp if new_lp
678
494
  @pb = new_pb if new_pb
679
495
 
496
+ # Recompute cached values when properties change
497
+ if properties_changed
498
+ @pb_mask = (1 << @pb) - 1
499
+ @pb_shift = 1 << @pb
500
+ @literal_mask = (0x100 << @lp) - (0x100 >> @lc)
501
+ end
502
+
680
503
  # Reset state machine (XZ Utils line 1053)
681
504
  # Always create a new state machine when resetting
682
505
  @state = SdkStateMachine.new
@@ -695,53 +518,31 @@ check_rc_finished: true)
695
518
  # - rc_reset sets range=UINT32_MAX, code=0, init_bytes_left=5
696
519
  # - This MUST happen during reset, not deferred to decode_stream
697
520
  # Reference: /Users/mulgogi/src/external/xz/src/liblzma/lzma/lzma_decoder.c:1061
698
- if @range_decoder
699
- @range_decoder.reset
700
- if ENV["LZMA_DEBUG_RESET"]
701
- warn "DEBUG reset[#{@decoder_id}]: Reset range_decoder, code=0x#{@range_decoder.code.to_s(16)}, init_bytes_remaining=#{@range_decoder.instance_variable_get(:@init_bytes_remaining)}"
702
- end
703
- end
521
+ @range_decoder&.reset
704
522
 
705
523
  # Reinitialize probability models (XZ Utils lines 1064-1082)
706
524
  # IMPORTANT: Use reset_models (reset in place) instead of init_models (create new)
707
525
  # for state reset only. Only create new models when properties change.
708
526
  if properties_changed
709
- if ENV["LZMA_DEBUG_RESET"]
710
- warn "DEBUG reset[#{@decoder_id}]: Properties changed, calling init_models (create new arrays)"
711
- end
712
527
  init_models
713
528
  else
714
- if ENV["LZMA_DEBUG_RESET"]
715
- warn "DEBUG reset[#{@decoder_id}]: No properties changed, calling reset_models (reset in place)"
716
- end
717
529
  reset_models
718
530
  end
719
531
 
720
532
  # Reinitialize coders (needed for pb changes)
721
533
  # Only recreate coders when properties have changed
722
- if properties_changed
723
- if ENV["LZMA_DEBUG_RESET"]
724
- warn "DEBUG reset[#{@decoder_id}]: Properties changed, calling init_coders (create new coders)"
725
- end
726
- init_coders
727
- elsif ENV["LZMA_DEBUG_RESET"]
728
- warn "DEBUG reset[#{@decoder_id}]: No properties changed, skipping init_coders (preserve existing coders)"
729
- end
534
+ init_coders if properties_changed
730
535
 
731
536
  # Reset dictionary position and full count (XZ Utils pattern)
732
537
  # Only reset if preserve_dict is false
733
538
  unless preserve_dict
734
539
  # Reinitialize dictionary buffer
735
540
  # XZ Utils allocates a new buffer for each dictionary reset
736
- @dict_buf = Array.new(@dict_size + LZ_DICT_INIT_POS, 0)
541
+ @dict_buf = ("\0" * (@dict_size + LZ_DICT_INIT_POS)).b
737
542
  @pos = LZ_DICT_INIT_POS
543
+ @dict_pos = LZ_DICT_INIT_POS
738
544
  @dict_full = 0
739
545
  @has_wrapped = false
740
- if ENV["LZMA_DEBUG_RESET"]
741
- warn "DEBUG reset after dict reset[#{@decoder_id}]: @pos=#{@pos}, @dict_full=#{@dict_full}, @dict_buf.size=#{@dict_buf.size}, @dict_buf.object_id=#{@dict_buf.object_id}, nil_count=#{@dict_buf.count(nil)}"
742
- # Verify buffer initialization by checking a few positions
743
- warn " Sample values: [576]=#{@dict_buf[576]}, [577]=#{@dict_buf[577]}, [578]=#{@dict_buf[578]}, [583]=#{@dict_buf[583]}"
744
- end
745
546
  end
746
547
 
747
548
  nil
@@ -799,12 +600,6 @@ check_rc_finished: true)
799
600
  #
800
601
  # @return [void]
801
602
  def prepare_state_reset
802
- # DEBUG: Trace when prepare_state_reset is called
803
- if ENV["LZMA_DEBUG"]
804
- XzUtilsDecoderDebug.debug_puts "\n=== prepare_state_reset called (decoder_id=#{@decoder_id}) ==="
805
- XzUtilsDecoderDebug.debug_puts " Before reset: rep0/1/2/3=(#{@rep0},#{@rep1},#{@rep2},#{@rep3})"
806
- end
807
-
808
603
  # Reset state machine (XZ Utils line 1053)
809
604
  @state = SdkStateMachine.new
810
605
 
@@ -814,18 +609,9 @@ check_rc_finished: true)
814
609
  @rep2 = 0
815
610
  @rep3 = 0
816
611
 
817
- # DEBUG: Show after reset
818
- if ENV["LZMA_DEBUG"]
819
- XzUtilsDecoderDebug.debug_puts " After reset: rep0/1/2/3=(#{@rep0},#{@rep1},#{@rep2},#{@rep3})"
820
- end
821
-
822
612
  # Reset probability models (XZ Utils init_temporals for control >= 0xA0)
823
613
  reset_models
824
614
 
825
- if ENV["LZMA_DEBUG"]
826
- XzUtilsDecoderDebug.debug_puts "=== end prepare_state_reset (range decoder will be reset in finish_state_reset) ==="
827
- end
828
-
829
615
  nil
830
616
  end
831
617
 
@@ -837,23 +623,12 @@ check_rc_finished: true)
837
623
  #
838
624
  # @return [void]
839
625
  def reset_state_machine_only
840
- # DEBUG: Trace when reset_state_machine_only is called
841
- if @dict_full && @dict_full >= 220 && @dict_full <= 230
842
- XzUtilsDecoderDebug.debug_puts "\n=== reset_state_machine_only called at dict_full=#{@dict_full} (decoder_id=#{@decoder_id}) ==="
843
- XzUtilsDecoderDebug.debug_puts " Before reset: rep0/1/2/3=(#{@rep0},#{@rep1},#{@rep2},#{@rep3})"
844
- end
845
-
846
626
  # Reset state machine only (XZ Utils line 1053)
847
627
  @state = SdkStateMachine.new
848
628
 
849
629
  # Reset probability models (XZ Utils init_temporals for control >= 0xA0)
850
630
  reset_models
851
631
 
852
- # DEBUG: Show after reset (note: rep distances are preserved)
853
- if @dict_full && @dict_full >= 220 && @dict_full <= 230
854
- XzUtilsDecoderDebug.debug_puts " After reset: rep0/1/2/3=(#{@rep0},#{@rep1},#{@rep2},#{@rep3}) (preserved)"
855
- end
856
-
857
632
  nil
858
633
  end
859
634
 
@@ -872,30 +647,7 @@ check_rc_finished: true)
872
647
  # Reset range decoder (XZ Utils rc_reset)
873
648
  # This reinitializes the range decoder for the new chunk
874
649
  # The reset will read 5 bytes from the input when decode_stream starts
875
- if @range_decoder
876
- if ENV["LZMA_DEBUG"]
877
- input_pos = begin
878
- @input.pos
879
- rescue StandardError
880
- "N/A"
881
- end
882
- input_size = begin
883
- @input.size
884
- rescue StandardError
885
- "N/A"
886
- end
887
- XzUtilsDecoderDebug.debug_puts "=== finish_state_reset: resetting range_decoder, input pos=#{input_pos}, size=#{input_size}"
888
- end
889
- @range_decoder.reset
890
- if ENV["LZMA_DEBUG"]
891
- input_pos_after = begin
892
- @input.pos
893
- rescue StandardError
894
- "N/A"
895
- end
896
- XzUtilsDecoderDebug.debug_puts "=== finish_state_reset: after reset, input pos=#{input_pos_after}, range_decoder.code=0x#{@range_decoder.code.to_s(16)}"
897
- end
898
- end
650
+ @range_decoder&.reset
899
651
  end
900
652
 
901
653
  def reset_state_only
@@ -931,45 +683,13 @@ check_rc_finished: true)
931
683
  def set_input(new_input)
932
684
  @input = new_input
933
685
 
934
- # DEBUG: Trace input stream contents
935
- if ENV.fetch("LZMA_DEBUG",
936
- nil) && @dict_full && @dict_full >= 220 && @dict_full <= 230
937
- puts "\n=== set_input at dict_full=#{@dict_full} ==="
938
- puts " new_input.size=#{new_input.size}"
939
- puts " new_input.pos=#{new_input.pos}"
940
- puts " new_input.class=#{new_input.class}"
941
-
942
- # Read first 10 bytes manually
943
- first_bytes = []
944
- 10.times do |_i|
945
- byte = new_input.getbyte
946
- break if byte.nil?
947
-
948
- first_bytes << byte
949
- end
950
- puts " First 10 bytes: #{first_bytes.map do |b|
951
- "0x#{b.to_s(16).upcase}"
952
- end.join(' ')}"
953
-
954
- new_input.rewind
955
- test_byte = new_input.getbyte
956
- puts " Test getbyte: 0x#{test_byte.to_s(16).upcase}" if test_byte
957
- new_input.rewind
958
- end
959
-
960
686
  # Create range decoder if it doesn't exist (first chunk)
961
687
  if @range_decoder.nil?
962
688
  @range_decoder = RangeDecoder.new(@input)
963
- if ENV["LZMA_DEBUG"]
964
- XzUtilsDecoderDebug.debug_puts "=== set_input: created NEW range_decoder, input has #{@input.size} bytes"
965
- end
966
689
  else
967
690
  # Update the range decoder's input stream to the new input
968
691
  # This is needed because RangeDecoder holds a reference to the stream
969
692
  @range_decoder.update_stream(@input)
970
- if ENV["LZMA_DEBUG"]
971
- XzUtilsDecoderDebug.debug_puts "=== set_input: reusing range_decoder, new input has #{@input.size} bytes, pos=#{@input.pos}"
972
- end
973
693
  end
974
694
  end
975
695
 
@@ -985,29 +705,30 @@ check_rc_finished: true)
985
705
  # @param data [String] Uncompressed data to add to dictionary
986
706
  # @return [void]
987
707
  def add_to_dictionary(data)
988
- if ENV["LZMA_DEBUG"]
989
- old_dict_full = @dict_full
990
- XzUtilsDecoderDebug.debug_puts "=== add_to_dictionary: adding #{data.bytesize} bytes to dictionary[#{@decoder_id}], current dict_full=#{@dict_full}, pos=#{@pos}"
991
- end
992
-
708
+ idx = @dict_pos
709
+ buf_end = @buf_end
993
710
  data.each_byte do |byte|
994
- @dict_buf[@pos] = byte
711
+ @dict_buf.setbyte(idx, byte)
995
712
  @pos += 1
713
+ idx += 1
714
+ idx = LZ_DICT_INIT_POS if idx >= buf_end
996
715
  end
716
+ @dict_pos = idx
997
717
 
998
718
  # Update dict_full to reflect new data
999
- @dict_full = @pos - LZ_DICT_INIT_POS
1000
-
1001
- # Check if we've reached the maximum dictionary size
1002
- if @dict_full >= @dict_size
1003
- @dict_full = @dict_size
1004
- end
1005
-
1006
- if ENV["LZMA_DEBUG"]
1007
- XzUtilsDecoderDebug.debug_puts "=== add_to_dictionary: after adding, dict_full=#{@dict_full} (was #{old_dict_full}), pos=#{@pos}"
719
+ unless @has_wrapped
720
+ @dict_full = @pos - LZ_DICT_INIT_POS
721
+ if @dict_full >= @dict_size
722
+ @has_wrapped = true
723
+ @dict_full = @dict_size
724
+ end
1008
725
  end
1009
726
  end
1010
727
 
728
+ # No-op: circular buffer has fixed size, no compaction needed.
729
+ # Kept for API compatibility — LZMA2 decoder calls this between chunks.
730
+ def compact_buffer; end
731
+
1011
732
  # Set uncompressed size for chunked decoding
1012
733
  #
1013
734
  # XZ Utils pattern (lzma2_decoder.c:140-141):
@@ -1020,10 +741,6 @@ check_rc_finished: true)
1020
741
  def set_uncompressed_size(size, allow_eopm: true)
1021
742
  @uncompressed_size = size
1022
743
  @allow_eopm = allow_eopm
1023
- # DEBUG: Track when uncompressed size is set
1024
- if ENV["LZMA_DEBUG"]
1025
- puts "DEBUG set_uncompressed_size: size=#{size}, @decoder_id=#{@decoder_id}, @dict_full=#{@dict_full}"
1026
- end
1027
744
  end
1028
745
 
1029
746
  private
@@ -1136,13 +853,6 @@ check_rc_finished: true)
1136
853
  @rep_length_coder = LengthCoder.new(pos_states)
1137
854
  @distance_coder = DistanceCoder.new(NUM_LEN_TO_POS_STATES)
1138
855
 
1139
- if ENV["TRACE_MODEL_INIT"]
1140
- puts "[XzUtilsDecoder.init] slot_encoders len_state=0 object_id=#{@distance_coder.instance_variable_get(:@slot_encoders)[0].object_id}"
1141
- puts "[XzUtilsDecoder.init] slot_encoders[0][1] object_id=#{@distance_coder.instance_variable_get(:@slot_encoders)[0][1].object_id}"
1142
- puts "[XzUtilsDecoder.init] is_match_models object_id=#{@is_match_models.object_id}"
1143
- puts "[XzUtilsDecoder.init] is_match_models[0] object_id=#{@is_match_models[0].object_id}"
1144
- end
1145
-
1146
856
  # Update probability model indices to match new pos_states
1147
857
  # This is critical when pb changes between chunks
1148
858
  @pos_states = pos_states
@@ -1174,202 +884,36 @@ check_rc_finished: true)
1174
884
  #
1175
885
  # @return [void]
1176
886
  def decode_literal
1177
- # DEBUG: Trace literals around position 224-227
1178
- old_dict_full = @dict_full
1179
-
1180
- # DEBUG: Track how many times we're called for each position
1181
- if ENV["LZMA_DEBUG_DECODE_LITERAL"]
1182
- caller_info = caller(1..1).first
1183
- XzUtilsDecoderDebug.debug_puts "DEBUG decode_literal[#{@decoder_id}]: pos=#{@pos}, dict_full=#{@dict_full}, from=#{caller_info.label}"
1184
- end
1185
-
1186
- # DEBUG: Check array integrity before decode
1187
- if ENV.fetch("LZMA_DEBUG_ARRAY",
1188
- nil) && @dict_full.positive? && @pos > 1
1189
- idx = @pos - 1
1190
- if @dict_buf[idx].nil?
1191
- raise "DEBUG before decode: @dict_buf[#{idx}] is nil! @pos=#{@pos}, @dict_full=#{@dict_full}, @dict_buf.size=#{@dict_buf.size}, nil_count=#{@dict_buf.count(nil)}"
1192
- end
1193
- end
1194
-
1195
887
  # Calculate literal state using SDK formula
1196
888
  lit_state = calculate_literal_state
1197
889
 
1198
- # DEBUG: Trace lit_state at position 61
1199
- if @dict_full == 61 && ENV["TRACE_LITERAL_61"]
1200
- XzUtilsDecoderDebug.debug_puts "=== CALC_LITERAL_STATE at dict_full=61 ==="
1201
- XzUtilsDecoderDebug.debug_puts " prev_byte=#{@dict_full.positive? ? @dict_buf[@pos - 1] : 0}"
1202
- XzUtilsDecoderDebug.debug_puts " lit_state=#{lit_state}"
1203
- XzUtilsDecoderDebug.debug_puts " lc=#{@lc}, lp=#{@lp}"
1204
- XzUtilsDecoderDebug.debug_puts " state.value=#{@state.value}"
1205
- XzUtilsDecoderDebug.debug_puts " use_matched_literal?=#{@state.use_matched_literal?}"
1206
- XzUtilsDecoderDebug.debug_puts " dict_full.positive?=#{@dict_full.positive?}"
1207
- puts
1208
- end
1209
-
1210
890
  # Decode literal (matched or unmatched)
1211
891
  # Check if dictionary has any valid bytes (XZ Utils: dict->full > 0)
1212
892
  if @state.use_matched_literal? && @dict_full.positive?
1213
- # DEBUG: Track which branch is taken
1214
- if @dict_full == 61 && ENV["TRACE_LITERAL_61"]
1215
- XzUtilsDecoderDebug.debug_puts " TAKING MATCHED LITERAL PATH"
1216
- XzUtilsDecoderDebug.debug_puts " rep0=#{@rep0}"
1217
- match_byte_pos_calc = LZ_DICT_INIT_POS + @dict_full - @rep0 - 1
1218
- XzUtilsDecoderDebug.debug_puts " match_byte_pos (calc)=#{match_byte_pos_calc}"
1219
- puts
1220
- end
1221
-
1222
893
  # Matched literal: use match byte from dictionary at distance rep0
1223
894
  # XZ Utils dict_get pattern: dict->buf[dict->pos - distance - 1]
1224
- # IMPORTANT: dict->pos in XZ Utils is the actual output position (dict->full),
1225
- # not the buffer position with offset!
1226
- # omnizip uses @pos for buffer position (includes LZ_DICT_INIT_POS offset)
1227
- # and @dict_full for actual output position (starts at 0)
1228
- # So we must convert: buffer_pos = LZ_DICT_INIT_POS + (output_pos - rep0 - 1)
1229
- match_byte_pos = LZ_DICT_INIT_POS + @dict_full - @rep0 - 1
1230
- match_byte = @dict_buf[match_byte_pos]
1231
- if ENV["LZMA_DEBUG"]
1232
- warn "DEBUG: matched literal - dict_full=#{@dict_full}, rep0=#{@rep0}, reading dict_buf[#{match_byte_pos}]=0x#{match_byte.to_s(16).upcase} ('#{match_byte.chr}'), lit_state=#{lit_state}, state=#{@state.value}"
1233
- end
895
+ match_byte = @dict_buf.getbyte(dict_index(@pos - @rep0 - 1))
1234
896
  byte = @literal_decoder.decode_matched(match_byte, lit_state, @lc,
1235
897
  @range_decoder, @literal_models)
1236
-
1237
- # DEBUG: Trace decoded byte at position 61
1238
- if @dict_full == 61 && ENV["TRACE_LITERAL_61"]
1239
- XzUtilsDecoderDebug.debug_puts " DECODED MATCHED LITERAL: 0x#{byte.to_s(16).upcase} ('#{byte.chr}')"
1240
- XzUtilsDecoderDebug.debug_puts " match_byte=0x#{match_byte.to_s(16).upcase} ('#{match_byte.chr}')"
1241
- puts
1242
- end
1243
898
  else
1244
899
  # Unmatched literal: simple 8-bit decoding
1245
- if @dict_full == 61 && ENV["TRACE_LITERAL_61"]
1246
- XzUtilsDecoderDebug.debug_puts " TAKING UNMATCHED LITERAL PATH"
1247
- puts
1248
- end
1249
-
1250
- if ENV["LZMA_DEBUG"]
1251
- warn "DEBUG: calling decode_unmatched - pos=#{@pos}, lit_state=#{lit_state}"
1252
- end
1253
900
  byte = @literal_decoder.decode_unmatched(lit_state, @lc,
1254
901
  @range_decoder, @literal_models)
1255
902
  end
1256
903
 
1257
- if ENV["LZMA_DEBUG"]
1258
- warn "DEBUG: decode_literal RETURNED - pos=#{@pos}, byte=#{'0x%02X' % byte} ('#{byte.chr}'), lit_state=#{lit_state}"
1259
- end
1260
-
1261
- # DEBUG: Check byte value at critical positions
1262
- if @dict_full == 256
1263
- XzUtilsDecoderDebug.debug_puts "DEBUG: About to store 257th byte (pos=#{@pos}, dict_full=#{@dict_full})"
1264
- XzUtilsDecoderDebug.debug_puts " byte.class=#{byte.class}"
1265
- XzUtilsDecoderDebug.debug_puts " byte=#{byte.inspect}"
1266
- XzUtilsDecoderDebug.debug_puts " byte.is_a?(Integer)=#{byte.is_a?(Integer)}"
1267
- if byte.is_a?(Integer)
1268
- XzUtilsDecoderDebug.debug_puts " byte value=#{byte}"
1269
- XzUtilsDecoderDebug.debug_puts " Expected byte value=0"
1270
- else
1271
- XzUtilsDecoderDebug.debug_puts " byte is not an Integer!"
1272
- XzUtilsDecoderDebug.debug_puts " byte.ord=#{byte.ord}"
1273
- end
1274
- end
1275
-
1276
- if ENV["LZMA_DEBUG"]
1277
- decoded_bytes = @dict_full.positive? ? @dict_buf[LZ_DICT_INIT_POS..].map(&:chr).join : ""
1278
- warn "DEBUG: decode_literal - pos=#{@pos}, byte=#{'0x%02X' % byte} ('#{byte.chr}'), state=#{@state.value}, dict_full=#{@dict_full}, decoded_so_far='#{decoded_bytes[-10..]}'"
1279
- end
1280
-
1281
- # DEBUG: Detailed trace around position 256
1282
- if ENV["LZMA_DEBUG"] && @dict_full.between?(230, 265)
1283
- expected = @dict_full % 256
1284
- match = byte == expected ? "✓" : "✗ MISMATCH!"
1285
- XzUtilsDecoderDebug.debug_puts " [LITERAL] dict_full=#{@dict_full}: 0x#{byte.to_s(16).upcase} (expected 0x#{expected.to_s(16).upcase}) #{match}"
1286
- if @dict_full == 233
1287
- XzUtilsDecoderDebug.debug_puts " DETAILED TRACE at dict_full=233 (pos=#{@pos}):"
1288
- XzUtilsDecoderDebug.debug_puts " byte=0x#{byte.to_s(16)} ('#{begin
1289
- byte.chr
1290
- rescue StandardError
1291
- '?'
1292
- end}')"
1293
- XzUtilsDecoderDebug.debug_puts " state.value=#{@state.value}, lit_state=#{lit_state}"
1294
- XzUtilsDecoderDebug.debug_puts " use_matched_literal?=#{@state.use_matched_literal?}"
1295
- prev_byte_val = @dict_full.positive? ? @dict_buf[@pos - 1] : "N/A"
1296
- XzUtilsDecoderDebug.debug_puts " prev_byte=#{prev_byte_val.inspect} (#{if prev_byte_val.is_a?(Integer)
1297
- "0x#{prev_byte_val.to_s(16)} ('#{begin
1298
- prev_byte_val.chr
1299
- rescue StandardError
1300
- '?'
1301
- end}')"
1302
- else
1303
- 'N/A'
1304
- end})"
1305
- XzUtilsDecoderDebug.debug_puts " range_decoder.range=0x#{@range_decoder.range.to_s(16)}, range_decoder.code=0x#{@range_decoder.code.to_s(16)}"
1306
- XzUtilsDecoderDebug.debug_puts " input.pos=#{@input.pos}, input.size=#{@input.size}"
1307
- end
1308
- if @dict_full == 256
1309
- XzUtilsDecoderDebug.debug_puts " pos=#{@pos}, lit_state=#{lit_state}, state.value=#{@state.value}"
1310
- XzUtilsDecoderDebug.debug_puts " use_matched_literal?=#{@state.use_matched_literal?}"
1311
- end
1312
- end
1313
-
1314
904
  # Update state and dictionary
1315
905
  # XZ Utils dict_put pattern:
1316
906
  # dict->buf[dict->pos++] = byte;
1317
907
  # if (!dict->has_wrapped)
1318
908
  # dict->full = dict->pos - LZ_DICT_INIT_POS;
1319
909
  @state.update_literal
1320
- warn "DEBUG: After update_literal - state=#{@state.value}" if ENV["LZMA_DEBUG"]
1321
910
 
1322
911
  # Write to dictionary buffer at current position
1323
912
  # XZ Utils dict_put pattern: dict->buf[dict->pos++] = byte;
1324
- # DEBUG: Check byte value at critical position
1325
- if @pos == 576 + 256
1326
- XzUtilsDecoderDebug.debug_puts "DEBUG: Storing byte at pos 832 (256th decoded byte)"
1327
- XzUtilsDecoderDebug.debug_puts " byte.class=#{byte.class}"
1328
- XzUtilsDecoderDebug.debug_puts " byte=#{byte}"
1329
- XzUtilsDecoderDebug.debug_puts " byte.ord=#{byte.is_a?(String) ? byte.ord : 'N/A (not a string)'}"
1330
- XzUtilsDecoderDebug.debug_puts " Integer value=#{byte.is_a?(Integer) ? byte : byte.ord}"
1331
- end
1332
- @dict_buf[@pos] = byte
1333
- # DEBUG: Track array size changes
1334
- if ENV["LZMA_DEBUG_ARRAY_WRITE"] && @dict_buf.size != (@dict_size + LZ_DICT_INIT_POS)
1335
- XzUtilsDecoderDebug.debug_puts "DEBUG: Array expanded! pos=#{@pos}, byte=#{byte}, old_size=#{@dict_buf.size - 1}, new_size=#{@dict_buf.size}, decoder_id=#{@decoder_id}"
1336
- XzUtilsDecoderDebug.debug_puts " Writing beyond original size caused expansion!"
1337
- end
1338
- if ENV["LZMA_DEBUG_ARRAY_WRITE"]
1339
- XzUtilsDecoderDebug.debug_puts "DEBUG write[#{@decoder_id}]: pos=#{@pos}, byte=#{byte}, dict_buf.size=#{@dict_buf.size}, dict_buf.object_id=#{@dict_buf.object_id}, nil_count=#{@dict_buf.count(nil)}"
1340
- end
1341
- if ENV["LZMA_DEBUG_ARRAY"]
1342
- # Verify the write succeeded
1343
- if @dict_buf[@pos] != byte
1344
- raise "DEBUG after write: @dict_buf[#{@pos}] = #{@dict_buf[@pos].inspect}, expected #{byte}! nil_count=#{@dict_buf.count(nil)}"
1345
- end
1346
- if @dict_full.positive? && @pos > LZ_DICT_INIT_POS && @dict_buf[@pos - 1].nil?
1347
- raise "DEBUG after write: @dict_buf[#{@pos - 1}] is nil! @pos=#{@pos}, @dict_full=#{@dict_full}"
1348
- end
1349
- end
913
+ @dict_buf.setbyte(@dict_pos, byte)
1350
914
  @pos += 1
1351
-
1352
- # ARM64 DEBUG: Trace first 20 bytes being written to dictionary
1353
- if ENV["TRACE_ARM64_BYTES"]
1354
- @arm64_trace ||= []
1355
- if @arm64_trace.size < 20
1356
- @arm64_trace << [@dict_full, @pos, byte.class,
1357
- byte.is_a?(Integer) ? byte : byte.ord, @dict_buf[@pos]]
1358
- if @arm64_trace.size == 20
1359
- # Dump the trace
1360
- puts "\n=== ARM64 BYTE TRACE (first 20 bytes) ==="
1361
- puts "Decoder ID: #{@decoder_id}"
1362
- @arm64_trace.each_with_index do |entry, i|
1363
- df, p, _, val, stored = entry
1364
- puts " [#{i + 1}] dict_full=#{df.to_s.rjust(6)}, pos=#{p.to_s.rjust(6)}, byte=#{val.to_s.rjust(3)} (0x#{val.to_s(16).upcase.rjust(
1365
- 2, '0'
1366
- )}) stored=#{stored.inspect}"
1367
- end
1368
- puts "=========================================\n"
1369
- $stderr.flush
1370
- end
1371
- end
1372
- end
915
+ @dict_pos += 1
916
+ @dict_pos = LZ_DICT_INIT_POS if @dict_pos >= @buf_end
1373
917
 
1374
918
  # Update dict_full (XZ Utils pattern)
1375
919
  # When dict_full reaches dict_size, the dictionary is full
@@ -1390,13 +934,6 @@ check_rc_finished: true)
1390
934
  if @uncompressed_size != 0xFFFFFFFFFFFFFFFF
1391
935
  @chunk_bytes_decoded += 1
1392
936
  end
1393
-
1394
- # DEBUG: Show literal decode for position 220-230
1395
- if old_dict_full.between?(220, 230)
1396
- XzUtilsDecoderDebug.debug_puts "\n=== decode_literal at dict_full=#{old_dict_full} ==="
1397
- XzUtilsDecoderDebug.debug_puts " Decoded: 0x#{byte.to_s(16)} ('#{byte.chr}')"
1398
- XzUtilsDecoderDebug.debug_puts " rep0/1/2/3=(#{@rep0},#{@rep1},#{@rep2},#{@rep3})"
1399
- end
1400
937
  end
1401
938
 
1402
939
  # Decode a match
@@ -1416,60 +953,18 @@ check_rc_finished: true)
1416
953
  #
1417
954
  # @return [Boolean] True if EOS marker detected, false otherwise
1418
955
  def decode_match
1419
- pos_state = @pos & ((1 << @pb) - 1)
1420
-
1421
- # DEEP DEBUG: Trace every detail at position 227
1422
- if ENV["LZMA_DEBUG"] && @dict_full == 227
1423
- puts "\n=== DEEP DEBUG at dict_full=227 ==="
1424
- puts " State: #{@state.value}, pos_state=#{pos_state}"
1425
- puts " Rep distances BEFORE: (#{@rep0},#{@rep1},#{@rep2},#{@rep3})"
1426
- puts " Range decoder: range=0x#{@range_decoder.range.to_s(16).upcase}, code=0x#{@range_decoder.code.to_s(16).upcase}"
1427
- input_buffer = @range_decoder.instance_variable_get(:@input)
1428
- puts " Input buffer: #{input_buffer ? input_buffer.size : 'nil'} bytes"
1429
- end
956
+ pos_state = @pos & @pb_mask
1430
957
 
1431
958
  # Decode is_rep bit
1432
959
  is_rep_model = @is_rep_models[@state.value]
1433
- if ENV["TRACE_IS_REP"]
1434
- range_val = @range_decoder.range
1435
- code_val = @range_decoder.code
1436
- puts "[XzUtilsDecoder.decode_match] Before is_rep: state.value=#{@state.value}"
1437
- puts " is_rep_model.object_id=#{is_rep_model.object_id}, prob=#{is_rep_model.probability}"
1438
- puts " range=#{range_val} (0x#{range_val.to_s(16)}), code=#{code_val} (0x#{code_val.to_s(16)})"
1439
- bound_calc = (range_val >> 11) * is_rep_model.probability
1440
- puts " bound=(#{range_val} >> 11) * #{is_rep_model.probability} = #{bound_calc} (0x#{bound_calc.to_s(16)})"
1441
- puts " code < bound? #{code_val < bound_calc}"
1442
- end
1443
960
  is_rep = @range_decoder.decode_bit(is_rep_model)
1444
961
 
1445
- if ENV["TRACE_IS_REP"]
1446
- range_val = @range_decoder.range
1447
- code_val = @range_decoder.code
1448
- puts "[XzUtilsDecoder.decode_match] Decoded is_rep=#{is_rep} with prob=#{is_rep_model.probability}"
1449
- puts " After is_rep: range=#{range_val} (0x#{range_val.to_s(16)}), code=#{code_val} (0x#{code_val.to_s(16)})"
1450
- end
1451
-
1452
- if ENV["LZMA_DEBUG"] && @dict_full == 227
1453
- puts " Decoded is_rep bit: #{is_rep} (#{@is_rep_models[@state.value].probability})"
1454
- puts " After is_rep: range=0x#{@range_decoder.range.to_s(16).upcase}, code=0x#{@range_decoder.code.to_s(16).upcase}"
1455
- end
1456
-
1457
- if ENV["LZMA_DEBUG"]
1458
- warn "DEBUG: decode_match START - is_rep=#{is_rep}, state.value=#{@state.value}, pos_state=#{pos_state}, rep0/1/2/3=(#{@rep0},#{@rep1},#{@rep2},#{@rep3})"
1459
- end
1460
-
1461
962
  if is_rep.zero?
1462
963
  # Regular match (not rep)
1463
- if ENV["LZMA_DEBUG"] && @dict_full.between?(220, 240)
1464
- puts "DEBUG pos #{@dict_full}: Regular match (not rep)"
1465
- end
1466
964
  # Return result from decode_regular_match (true if EOS marker detected)
1467
965
  return true if decode_regular_match(pos_state)
1468
966
  else
1469
967
  # Rep match - decode which rep distance to use
1470
- if ENV["LZMA_DEBUG"] && @dict_full.between?(220, 240)
1471
- puts "DEBUG pos #{@dict_full}: Rep match (is_rep=1)"
1472
- end
1473
968
  decode_rep_match(pos_state)
1474
969
  end
1475
970
 
@@ -1488,14 +983,8 @@ check_rc_finished: true)
1488
983
  # @param pos_state [Integer] Position state
1489
984
  # @return [Boolean] True if EOS marker detected, false otherwise
1490
985
  def decode_regular_match(pos_state)
1491
- # DEBUG: Trace matches around dict_full = 60-63
1492
- old_dict_full = @dict_full
1493
- old_rep0 = @rep0
1494
- old_state = @state.value
1495
-
1496
986
  # Decode match length
1497
- length_encoded = @length_coder.decode(@range_decoder,
1498
- pos_state)
987
+ length_encoded = @length_coder.decode(@range_decoder, pos_state)
1499
988
  length = length_encoded + MATCH_LEN_MIN
1500
989
 
1501
990
  # Calculate length state for distance decoding
@@ -1508,54 +997,11 @@ check_rc_finished: true)
1508
997
  NUM_LEN_TO_POS_STATES - 1
1509
998
  end
1510
999
 
1511
- # DEBUG: Show bytes being copied
1512
- if old_dict_full.between?(210, 230) || ENV["LZMA_DEBUG_DISTANCE"]
1513
- XzUtilsDecoderDebug.debug_puts "\n=== decode_regular_match at dict_full=#{old_dict_full} ===" if old_dict_full.between?(
1514
- 210, 230
1515
- )
1516
- puts "[DISTANCE_DECODER] decode_regular_match at dict_full=#{old_dict_full}" if ENV["LZMA_DEBUG_DISTANCE"]
1517
- XzUtilsDecoderDebug.debug_puts " pos_state=#{pos_state}" if old_dict_full.between?(
1518
- 210, 230
1519
- )
1520
- puts "[DISTANCE_DECODER] pos_state=#{pos_state}" if ENV["LZMA_DEBUG_DISTANCE"]
1521
- XzUtilsDecoderDebug.debug_puts " state=#{old_state}" if old_dict_full.between?(
1522
- 210, 230
1523
- )
1524
- puts "[DISTANCE_DECODER] state=#{old_state}" if ENV["LZMA_DEBUG_DISTANCE"]
1525
- XzUtilsDecoderDebug.debug_puts " length_encoded=#{length_encoded} length=#{length}" if old_dict_full.between?(
1526
- 210, 230
1527
- )
1528
- puts "[DISTANCE_DECODER] length_encoded=#{length_encoded} length=#{length}" if ENV["LZMA_DEBUG_DISTANCE"]
1529
- XzUtilsDecoderDebug.debug_puts " len_state=#{len_state}" if old_dict_full.between?(
1530
- 210, 230
1531
- )
1532
- puts "[DISTANCE_DECODER] len_state=#{len_state}" if ENV["LZMA_DEBUG_DISTANCE"]
1533
- XzUtilsDecoderDebug.debug_puts " rep0_before=#{old_rep0}" if old_dict_full.between?(
1534
- 210, 230
1535
- )
1536
- puts "[DISTANCE_DECODER] rep0_before=#{old_rep0}" if ENV["LZMA_DEBUG_DISTANCE"]
1537
- end
1538
-
1539
- if ENV["LZMA_DEBUG"] && old_dict_full.between?(220, 230)
1540
- puts "DEBUG decode_regular_match at dict_full=#{old_dict_full}: length=#{length}"
1541
- end
1542
-
1543
1000
  # Decode match distance
1544
1001
  # XZ Utils stores distance in rep0 without +1
1545
1002
  # The distance coder returns 0-based distance
1546
1003
  rep0 = @distance_coder.decode(@range_decoder, len_state)
1547
1004
 
1548
- # DEBUG
1549
- if (ENV.fetch("LZMA_DEBUG",
1550
- nil) && old_dict_full.between?(210,
1551
- 230)) || old_dict_full == 293
1552
- puts " rep0_decoded=#{rep0} (distance = #{rep0})"
1553
- puts " buffer_back calculation: back=#{@dict_full - rep0 - 1}"
1554
- end
1555
- if ENV["LZMA_DEBUG"] && rep0 > 100000
1556
- puts " [LARGE_DISTANCE at dict_full=#{old_dict_full}] rep0=#{rep0}, call count=#{$distance_decode_count || 'unknown'}"
1557
- end
1558
-
1559
1005
  # Check for SDK EOS marker FIRST (before validation)
1560
1006
  # EOS marker: rep0 == UINT32_MAX (0xFFFFFFFF)
1561
1007
  # XZ Utils checks: if (rep0 == UINT32_MAX) goto eopm;
@@ -1582,22 +1028,13 @@ check_rc_finished: true)
1582
1028
  end
1583
1029
  end
1584
1030
 
1585
- # Validate distance using XZ Utils dict_is_distance_valid pattern
1586
- # XZ Utils: lzma_decoder.c:876 - dict_is_distance_valid(&dict, rep0)
1587
- # See lz_decoder.h:194-198: return dict->full > distance;
1588
- # XZ Utils dict->full = dict->pos - LZ_DICT_INIT_POS (same as our @dict_full)
1589
- # The distance is valid if: dict->full > distance
1590
- unless @dict_full > rep0
1031
+ # Validate distance: ensure we have enough bytes in the buffer.
1032
+ # In this linear buffer model, @pos grows unbounded across chunks while
1033
+ # @dict_full is clamped at @dict_size. Use actual bytes written for validation.
1034
+ actual_bytes_written = @pos - LZ_DICT_INIT_POS
1035
+ unless actual_bytes_written > rep0
1591
1036
  raise Omnizip::DecompressionError,
1592
- "Invalid distance: #{rep0} (dict_full: #{@dict_full})"
1593
- end
1594
-
1595
- # Additional validation: ensure distance doesn't exceed absolute dictionary size
1596
- # Only validate against absolute dictionary size to prevent buffer overflow
1597
- # The wrapping logic (below) handles rep0 >= @dict_full cases
1598
- if rep0 > @dict_size + @dict_full
1599
- raise Omnizip::DecompressionError,
1600
- "Invalid distance: #{rep0} exceeds maximum possible (dict_size=#{@dict_size}, dict_full=#{@dict_full})"
1037
+ "Invalid distance: #{rep0} (bytes_written: #{actual_bytes_written})"
1601
1038
  end
1602
1039
 
1603
1040
  # IMPORTANT: Limit match length to not exceed uncompressed_size
@@ -1609,9 +1046,6 @@ check_rc_finished: true)
1609
1046
  # @uncompressed_size is the target for THIS chunk (not cumulative)
1610
1047
  remaining = @uncompressed_size - @chunk_bytes_decoded
1611
1048
  if length > remaining
1612
- if ENV["LZMA_DEBUG"] || remaining <= 5
1613
- puts "DEBUG: Limiting match length from #{length} to #{remaining} (chunk_bytes_decoded=#{@chunk_bytes_decoded}, uncompressed_size=#{@uncompressed_size}, dict_full=#{@dict_full})"
1614
- end
1615
1049
  length = remaining
1616
1050
  end
1617
1051
  end
@@ -1627,75 +1061,28 @@ check_rc_finished: true)
1627
1061
  #
1628
1062
  # dict->size in XZ Utils = dict_size + 2 * LZ_DICT_REPEAT_MAX
1629
1063
  # Our dict_buf size = @dict_size + LZ_DICT_INIT_POS = @dict_size + 2 * LZ_DICT_REPEAT_MAX
1630
- back = @dict_full - rep0 - 1
1631
- if rep0 >= @dict_full
1632
- # Distance wraps to the end of the dictionary buffer
1633
- # XZ Utils: back += dict->size - LZ_DICT_REPEAT_MAX;
1634
- # Our dict_buf size = @dict_size + LZ_DICT_INIT_POS = @dict_size + 2 * LZ_DICT_REPEAT_MAX
1635
- # So: back += (@dict_size + 2 * LZ_DICT_REPEAT_MAX) - LZ_DICT_REPEAT_MAX
1636
- # = back + @dict_size + LZ_DICT_REPEAT_MAX
1637
- back += @dict_size + LZ_DICT_REPEAT_MAX
1638
- end
1639
- # Convert to buffer position (add LZ_DICT_INIT_POS offset)
1640
- buffer_back = back + LZ_DICT_INIT_POS
1641
-
1642
- # DEBUG: Show buffer position for position 217
1643
- if old_dict_full.between?(210, 230)
1644
- XzUtilsDecoderDebug.debug_puts " buffer_back=#{buffer_back}, back=#{back}"
1645
- bytes_at_back = @dict_buf[buffer_back, 3]
1646
- bytes_hex = if bytes_at_back.is_a?(String)
1647
- bytes_at_back.bytes.map do |b|
1648
- "%02x" % b
1649
- end.join(" ")
1650
- else
1651
- bytes_at_back.map do |b|
1652
- "%02x" % b
1653
- end.join(" ")
1654
- end
1655
- XzUtilsDecoderDebug.debug_puts " First 3 bytes at buffer_back: #{bytes_hex} (#{bytes_at_back.inspect})"
1656
- end
1657
-
1658
- if ENV["LZMA_DEBUG"]
1659
- b0 = @dict_buf[buffer_back]
1660
- b1 = @dict_buf[buffer_back + 1]
1661
- b2 = @dict_buf[buffer_back + 2]
1662
- b0_str = b0 ? "0x#{b0.to_s(16).upcase}" : "nil"
1663
- b1_str = b1 ? "0x#{b1.to_s(16).upcase}" : "nil"
1664
- b2_str = b2 ? "0x#{b2.to_s(16).upcase}" : "nil"
1665
- b0_chr = b0 ? "'#{b0.chr}'" : "nil"
1666
- b1_chr = b1 ? "'#{b1.chr}'" : "nil"
1667
- b2_chr = b2 ? "'#{b2.chr}'" : "nil"
1668
- warn "DEBUG: copy from buffer_back=#{buffer_back} (distance #{rep0}): #{b0_str} (#{b0_chr}) #{b1_str} (#{b1_chr}) #{b2_str} (#{b2_chr})"
1669
- warn "DEBUG: pos_before=#{@pos} (output #{@pos - LZ_DICT_INIT_POS}), len=#{length}, pos_after=#{@pos + length} (output #{@pos + length - LZ_DICT_INIT_POS})"
1670
- # Show what the dictionary contains at key positions (simplified)
1671
- warn "DEBUG: dict_buf size=#{@dict_buf.size}, allocated=#{@dict_size + 608}"
1672
- end
1064
+ # Linear buffer: use @pos directly for back reference
1065
+ # @pos always points to the next write position, so @pos - rep0 - 1
1066
+ # gives the correct source position for the match copy
1067
+ buffer_back = @pos - rep0 - 1
1673
1068
 
1674
1069
  # Copy bytes from dictionary and extend buffer as needed
1675
1070
  # XZ Utils dict_repeat pattern: dict->buf[dict->pos++] = dict->buf[back++]
1676
- if ENV["LZMA_DEBUG"] && old_dict_full.between?(220, 260)
1677
- puts " Copying #{length} bytes from buffer_back=#{buffer_back} to @pos=#{@pos}, dict_full=#{@dict_full}"
1678
- puts " Source bytes: #{@dict_buf[buffer_back, length].inspect}"
1679
- puts " First 5 target bytes before copy: #{@dict_buf[@pos,
1680
- 5].inspect}"
1681
- end
1682
- length.times do |i|
1683
- byte = @dict_buf[buffer_back + i]
1684
- if ENV["LZMA_DEBUG"]
1685
- warn "DEBUG: copy iteration #{i}: reading dict_buf[#{buffer_back + i}]=0x#{byte.to_s(16).upcase} ('#{byte.chr}'), writing to dict_buf[#{@pos + i}]"
1686
- end
1687
- @dict_buf[@pos + i] = byte
1688
- if ENV["LZMA_DEBUG_ARRAY_WRITE"] && @dict_buf.size != (@dict_size + LZ_DICT_INIT_POS)
1689
- XzUtilsDecoderDebug.debug_puts "DEBUG: Array expanded during copy! write_pos=#{@pos + i}, byte=#{byte}, old_size=#{@dict_buf.size - 1}, new_size=#{@dict_buf.size}, decoder_id=#{@decoder_id}"
1690
- end
1691
- end
1692
- if ENV["LZMA_DEBUG"] && old_dict_full.between?(220, 230)
1693
- puts " After copy: #{@dict_buf[@pos, length].inspect}"
1694
- end
1071
+ src_idx = dict_index(buffer_back)
1072
+ dst_idx = @dict_pos
1073
+ buf_end = @buf_end
1074
+ length.times do
1075
+ byte = @dict_buf.getbyte(src_idx)
1076
+ @dict_buf.setbyte(dst_idx, byte)
1077
+ src_idx += 1
1078
+ src_idx = LZ_DICT_INIT_POS if src_idx >= buf_end
1079
+ dst_idx += 1
1080
+ dst_idx = LZ_DICT_INIT_POS if dst_idx >= buf_end
1081
+ end
1082
+ @dict_pos = dst_idx
1695
1083
 
1696
1084
  # Update state and position
1697
1085
  @state.update_match
1698
- warn "DEBUG: After update_match - state=#{@state.value}" if ENV["LZMA_DEBUG"]
1699
1086
  @pos += length
1700
1087
 
1701
1088
  # Update dict_full (XZ Utils pattern)
@@ -1721,45 +1108,11 @@ check_rc_finished: true)
1721
1108
  # Update rep distances - rotate and set new rep0
1722
1109
  # SDK rotation: rep3←rep2, rep2←rep1, rep1←rep0, rep0←rep0
1723
1110
  # XZ Utils stores the actual distance in rep0 (no +1)
1724
- if ENV["LZMA_DEBUG"]
1725
- warn "DEBUG: Before rotation - rep0/1/2/3=(#{@rep0},#{@rep1},#{@rep2},#{@rep3}), new distance=#{rep0}"
1726
- end
1727
-
1728
- # DEBUG: Trace rep rotation for position 224
1729
- if ENV["LZMA_DEBUG"] && old_dict_full.between?(220, 230)
1730
- puts "\n=== Rep rotation after match at dict_full=#{old_dict_full} ==="
1731
- puts " Before: rep0/1/2/3=(#{@rep0},#{@rep1},#{@rep2},#{@rep3})"
1732
- puts " Setting rep0 to: #{rep0.inspect}"
1733
- end
1734
-
1735
1111
  @rep3 = @rep2
1736
1112
  @rep2 = @rep1
1737
1113
  @rep1 = @rep0
1738
1114
  @rep0 = rep0
1739
1115
 
1740
- if ENV["LZMA_DEBUG"]
1741
- warn "DEBUG: After rotation - rep0/1/2/3=(#{@rep0},#{@rep1},#{@rep2},#{@rep3})"
1742
- end
1743
-
1744
- # DEBUG: Show final rep values
1745
- if ENV["LZMA_DEBUG"] && old_dict_full.between?(220, 230)
1746
- puts " After: rep0/1/2/3=(#{@rep0},#{@rep1},#{@rep2},#{@rep3})"
1747
- end
1748
-
1749
- # DEBUG: Verify rep0 is actually set
1750
- if old_dict_full.between?(220, 230)
1751
- actual_rep0 = @rep0
1752
- XzUtilsDecoderDebug.debug_puts " VERIFICATION: @rep0=#{actual_rep0.inspect}, @rep0.object_id=#{@rep0.object_id}"
1753
- end
1754
-
1755
- # DEBUG: Trace range/code state after match at dict_full 56-62
1756
- if ENV.fetch("LZMA_DEBUG",
1757
- nil) && old_dict_full >= 56 && old_dict_full <= 62
1758
- range_after = @range_decoder.instance_variable_get(:@range)
1759
- code_after = @range_decoder.instance_variable_get(:@code)
1760
- XzUtilsDecoderDebug.debug_puts " AFTER match (dict_full #{old_dict_full}→#{@dict_full}): range=0x#{range_after.to_s(16).upcase}, code=0x#{code_after.to_s(16).upcase}"
1761
- end
1762
-
1763
1116
  false # Not EOS, continue decoding
1764
1117
  end
1765
1118
 
@@ -1777,49 +1130,16 @@ check_rc_finished: true)
1777
1130
  # @param pos_state [Integer] Position state
1778
1131
  # @return [Boolean] Always false (rep matches are never EOS)
1779
1132
  def decode_rep_match(pos_state)
1780
- # DEBUG: Trace rep matches around position 217
1781
- old_dict_full = @dict_full
1782
- old_rep0 = @rep0
1783
-
1784
- # DEBUG: Show rep distances at the start
1785
- if ENV["LZMA_DEBUG"]
1786
- warn "DEBUG: decode_rep_match START[#{@decoder_id}] - rep0/1/2/3=(#{@rep0},#{@rep1},#{@rep2},#{@rep3})"
1787
- end
1788
-
1789
- # DEBUG: Trace rep matches around position 227
1790
- if old_dict_full.between?(220, 230)
1791
- XzUtilsDecoderDebug.debug_puts "\n=== decode_rep_match at dict_full=#{old_dict_full} (decoder_id=#{@decoder_id}) ==="
1792
- XzUtilsDecoderDebug.debug_puts " At START: rep0/1/2/3=(#{@rep0},#{@rep1},#{@rep2},#{@rep3})"
1793
- XzUtilsDecoderDebug.debug_puts " old_rep0=#{old_rep0} (captured @rep0)"
1794
- XzUtilsDecoderDebug.debug_puts " @rep0.object_id=#{@rep0.object_id}"
1795
- end
1796
-
1797
1133
  # Decode which rep distance to use
1798
1134
  is_rep0 = @range_decoder.decode_bit(@is_rep0_models[@state.value])
1799
1135
 
1800
- puts "DEBUG rep match selection at dict_full=#{@dict_full}: is_rep0=#{is_rep0}, rep0/1/2/3 before=(#{@rep0},#{@rep1},#{@rep2},#{@rep3})" if @dict_full.between?(
1801
- 220, 230
1802
- )
1803
- puts " state.value=#{@state.value}, pos_state=#{pos_state}, model_index=#{(@state.value * (1 << @pb)) + pos_state}" if @dict_full.between?(
1804
- 220, 230
1805
- )
1806
-
1807
- if ENV["LZMA_DEBUG"]
1808
- warn "DEBUG: decode_rep_match - is_rep0=#{is_rep0}"
1809
- end
1810
-
1811
1136
  if is_rep0.zero?
1812
1137
  # Use rep0
1813
- puts "DEBUG rep match using rep0" if @dict_full.between?(220, 230)
1814
1138
  # XZ Utils: is_rep0_long[state][pos_state] where the array size is NUM_STATES * (1 << pb)
1815
1139
  is_rep0_long = @range_decoder.decode_bit(
1816
- @is_rep0_long_models[(@state.value * (1 << @pb)) + pos_state],
1140
+ @is_rep0_long_models[(@state.value * @pb_shift) + pos_state],
1817
1141
  )
1818
1142
 
1819
- if ENV["LZMA_DEBUG"]
1820
- warn "DEBUG: decode_rep_match - is_rep0_long=#{is_rep0_long}"
1821
- end
1822
-
1823
1143
  if is_rep0_long.zero?
1824
1144
  # Short rep (length=1)
1825
1145
  length = 1
@@ -1832,10 +1152,6 @@ check_rc_finished: true)
1832
1152
  end
1833
1153
  else
1834
1154
  # Not rep0, check rep1/rep2/rep3
1835
- puts "DEBUG rep match NOT using rep0 (is_rep0=#{is_rep0})" if @dict_full.between?(
1836
- 220, 230
1837
- )
1838
-
1839
1155
  is_rep1 = @range_decoder.decode_bit(@is_rep1_models[@state.value])
1840
1156
 
1841
1157
  if is_rep1.zero?
@@ -1880,27 +1196,12 @@ check_rc_finished: true)
1880
1196
  # XZ Utils stores distances without +1 offset
1881
1197
  distance = @rep0
1882
1198
 
1883
- puts "DEBUG rep match after rotation: dict_full=#{old_dict_full}, distance=#{distance}, rep0/1/2/3=(#{@rep0},#{@rep1},#{@rep2},#{@rep3})" if old_dict_full.between?(
1884
- 220, 230
1885
- )
1886
-
1887
- # DEBUG: Trace rep matches around position 217
1888
- if old_dict_full.between?(210, 230)
1889
- XzUtilsDecoderDebug.debug_puts "\n=== decode_rep_match at dict_full=#{old_dict_full} ==="
1890
- XzUtilsDecoderDebug.debug_puts " old_rep0=#{old_rep0}, new rep0=#{@rep0} (distance=#{distance})"
1891
- XzUtilsDecoderDebug.debug_puts " pos_state=#{pos_state}"
1892
- end
1893
-
1894
- if ENV["LZMA_DEBUG"]
1895
- warn "DEBUG: decode_rep_match - length=#{length}, distance=#{distance}, dict_full=#{@dict_full}, rep0/1/2/3=(#{@rep0},#{@rep1},#{@rep2},#{@rep3})"
1896
- end
1897
-
1898
- # Validate distance using XZ Utils dict_is_distance_valid pattern
1899
- # See lz_decoder.h:194-198: return dict->full > distance;
1900
- # Note: distance=0 is valid (means copy from position 0, the first byte)
1901
- # XZ Utils allows distance=0 as long as dict->full > 0 (at least one byte available)
1902
- unless @dict_full > distance
1903
- raise "Invalid rep distance: #{distance} (dict_full: #{@dict_full})"
1199
+ # Validate distance: ensure we have enough bytes in the buffer.
1200
+ # In this linear buffer model, @pos grows unbounded across chunks while
1201
+ # @dict_full is clamped at @dict_size. Use actual bytes written for validation.
1202
+ actual_bytes_written = @pos - LZ_DICT_INIT_POS
1203
+ unless actual_bytes_written > distance
1204
+ raise "Invalid rep distance: #{distance} (bytes_written: #{actual_bytes_written})"
1904
1205
  end
1905
1206
 
1906
1207
  # IMPORTANT: Limit match length to not exceed uncompressed_size
@@ -1912,9 +1213,6 @@ check_rc_finished: true)
1912
1213
  # @uncompressed_size is the target for THIS chunk (not cumulative)
1913
1214
  remaining = @uncompressed_size - @chunk_bytes_decoded
1914
1215
  if length > remaining
1915
- if ENV["LZMA_DEBUG"] || remaining <= 5
1916
- puts "DEBUG REP: Limiting rep match length from #{length} to #{remaining} (chunk_bytes_decoded=#{@chunk_bytes_decoded}, uncompressed_size=#{@uncompressed_size}, dict_full=#{@dict_full})"
1917
- end
1918
1216
  length = remaining
1919
1217
  end
1920
1218
  end
@@ -1928,79 +1226,23 @@ check_rc_finished: true)
1928
1226
  #
1929
1227
  # dict->size in XZ Utils = dict_size + 2 * LZ_DICT_REPEAT_MAX
1930
1228
  # Our dict_buf size = @dict_size + LZ_DICT_INIT_POS = @dict_size + 2 * LZ_DICT_REPEAT_MAX
1931
- back = @dict_full - distance - 1
1932
- old_back = back
1933
- if distance >= @dict_full
1934
- # Distance wraps to the end of the dictionary buffer
1935
- # XZ Utils: back += dict->size - LZ_DICT_REPEAT_MAX;
1936
- # Our dict_buf size = @dict_size + LZ_DICT_INIT_POS = @dict_size + 2 * LZ_DICT_REPEAT_MAX
1937
- # So: back += (@dict_size + 2 * LZ_DICT_REPEAT_MAX) - LZ_DICT_REPEAT_MAX
1938
- # = back + @dict_size + LZ_DICT_REPEAT_MAX
1939
- back += @dict_size + LZ_DICT_REPEAT_MAX
1940
- end
1941
- # Convert to buffer position (add LZ_DICT_INIT_POS offset)
1942
- buffer_back = back + LZ_DICT_INIT_POS
1943
-
1944
- puts "DEBUG rep match copy at dict_full=#{@dict_full}: @dict_full=#{@dict_full}, distance=#{distance}, back=#{back}, buffer_back=#{buffer_back}" if @dict_full.between?(
1945
- 220, 230
1946
- )
1947
-
1948
- # DEBUG: Show back calculation for position 217
1949
- if old_dict_full.between?(210, 230)
1950
- XzUtilsDecoderDebug.debug_puts " back calculation: @dict_full=#{@dict_full}, distance=#{distance}"
1951
- XzUtilsDecoderDebug.debug_puts " back=#{old_back}, wrapped_back=#{back}, buffer_back=#{buffer_back}"
1952
- bytes_at_back = @dict_buf[buffer_back, 3]
1953
- bytes_hex = if bytes_at_back.is_a?(String)
1954
- bytes_at_back.bytes.map do |b|
1955
- "%02x" % b
1956
- end.join(" ")
1957
- else
1958
- [bytes_at_back].flatten.map do |b|
1959
- "%02x" % b
1960
- end.join(" ")
1961
- end
1962
- XzUtilsDecoderDebug.debug_puts " First 3 bytes at buffer_back: #{bytes_hex} (#{bytes_at_back.inspect})"
1963
- end
1229
+ # Linear buffer: use @pos directly for back reference
1230
+ buffer_back = @pos - distance - 1
1964
1231
 
1965
- # Copy bytes from dictionary and extend buffer as needed
1232
+ # Copy bytes from dictionary
1966
1233
  # XZ Utils dict_repeat pattern: dict->buf[dict->pos++] = dict->buf[back++]
1967
- if old_dict_full.between?(250, 260)
1968
- source_val = @dict_buf[@pos - 1]
1969
- puts " Rep match copy at dict_full=#{@dict_full}: length=#{length}, distance=#{distance}, @pos=#{@pos} (will write to #{@pos}...#{@pos + length - 1})"
1970
- puts " Reading from @pos-1=#{@pos - 1}, source byte = #{source_val} (0x#{source_val.to_s(16)} '#{begin
1971
- source_val.chr
1972
- rescue StandardError
1973
- '?'
1974
- end}')"
1975
- puts " Before copy: @dict_buf[#{@pos}...#{@pos + length - 1}] = #{@dict_buf[@pos,
1976
- length].inspect}"
1977
- end
1978
- length.times do |i|
1979
- byte = @dict_buf[buffer_back + i]
1980
- if @dict_full == 227 && i.zero?
1981
- puts "DEBUG dict_copy at dict_full=227, i=0:"
1982
- puts " buffer_back=#{buffer_back}, byte=#{byte} ('#{byte.chr}')"
1983
- puts " Writing to @pos=#{@pos + i}"
1984
- puts " dict_buf[buffer_back...buffer_back+10] = #{@dict_buf[buffer_back,
1985
- 10].inspect}"
1986
- # DEBUG: Check if buffer_back+1 has the correct byte
1987
- puts " dict_buf[buffer_back+1=#{buffer_back + 1}] = #{@dict_buf[buffer_back + 1].inspect} ('#{begin
1988
- @dict_buf[buffer_back + 1].chr
1989
- rescue StandardError
1990
- '?'
1991
- end}')"
1992
- prev_5 = if buffer_back > 4
1993
- @dict_buf[(buffer_back - 5)..(buffer_back - 1)].map do |b|
1994
- "0x#{b.to_s(16).upcase} (#{b.chr})"
1995
- end.join(", ")
1996
- else
1997
- "N/A"
1998
- end
1999
- puts " Previous 5 bytes: [#{prev_5}]"
2000
- puts " Current dict_full=#{@dict_full}, @pos=#{@pos}"
2001
- end
2002
- @dict_buf[@pos + i] = byte
2003
- end
1234
+ src_idx = dict_index(buffer_back)
1235
+ dst_idx = @dict_pos
1236
+ buf_end = @buf_end
1237
+ length.times do
1238
+ byte = @dict_buf.getbyte(src_idx)
1239
+ @dict_buf.setbyte(dst_idx, byte)
1240
+ src_idx += 1
1241
+ src_idx = LZ_DICT_INIT_POS if src_idx >= buf_end
1242
+ dst_idx += 1
1243
+ dst_idx = LZ_DICT_INIT_POS if dst_idx >= buf_end
1244
+ end
1245
+ @dict_pos = dst_idx
2004
1246
 
2005
1247
  # Update position
2006
1248
  @pos += length
@@ -2050,40 +1292,19 @@ check_rc_finished: true)
2050
1292
  def calculate_literal_state
2051
1293
  # XZ Utils dict_get0 pattern: dict->buf[dict->pos - 1]
2052
1294
  # For array buffer, use @pos - 1 as index
2053
-
2054
- # DEBUG: Check buffer state before access
2055
- if ENV["LZMA_DEBUG_CALC_STATE"] && @dict_full == 8
2056
- XzUtilsDecoderDebug.debug_puts "DEBUG before calc_state[#{@decoder_id}]: pos=#{@pos}, dict_full=#{@dict_full}"
2057
- XzUtilsDecoderDebug.debug_puts " @dict_buf.object_id=#{@dict_buf.object_id}, size=#{@dict_buf.size}"
2058
- XzUtilsDecoderDebug.debug_puts " Accessing index #{@pos - 1}: value=#{@dict_buf[@pos - 1].inspect}"
2059
- end
2060
-
2061
- prev_byte = @dict_full.positive? ? @dict_buf[@pos - 1] : 0
2062
-
2063
- # Safeguard: if prev_byte is nil, use 0 and log detailed diagnostics
2064
- # This can happen if the buffer was not properly initialized or we're accessing the wrong buffer
2065
- if prev_byte.nil?
2066
- if ENV["LZMA_DEBUG_NIL_BYTE"]
2067
- raise "DEBUG: prev_byte is nil! decoder_id=#{@decoder_id}, @pos=#{@pos}, @dict_full=#{@dict_full}, @dict_buf.size=#{@dict_buf&.size || 'nil'}, accessing index #{@pos - 1}, nil_count=#{@dict_buf&.count(nil) || 'N/A'}, @dict_buf.object_id=#{@dict_buf&.object_id || 'nil'}"
2068
- end
2069
-
2070
- prev_byte = 0
2071
- end
2072
-
2073
- if ENV["LZMA_DEBUG_CALC_STATE"]
2074
- XzUtilsDecoderDebug.debug_puts "DEBUG calc_state[#{@decoder_id}]: pos=#{@pos}, dict_full=#{@dict_full}, @dict_buf.object_id=#{@dict_buf.object_id}, prev_byte=#{prev_byte}"
2075
- end
2076
-
2077
- # XZ Utils formula from lzma_common.h:literal_mask_calc
2078
- # literal_mask = (UINT32_C(0x100) << (lp)) - (UINT32_C(0x100) >> (lc))
2079
- # For lc=3, lp=0: (256 << 0) - (256 >> 3) = 256 - 32 = 224 (0xE0)
2080
- literal_mask = (0x100 << @lp) - (0x100 >> @lc)
2081
-
2082
- # Combine dict_full (actual decoded position) and prev_byte, then apply mask
2083
- # IMPORTANT: XZ Utils uses dict.pos (which starts at 0 and increments)
2084
- # omnizip's @pos starts at LZ_DICT_INIT_POS (576), so we use @dict_full instead
2085
- # This ensures we match XZ Utils's literal state calculation exactly
2086
- (((@dict_full << 8) + prev_byte) & literal_mask)
1295
+ prev_idx = @dict_pos == LZ_DICT_INIT_POS ? @buf_end - 1 : @dict_pos - 1
1296
+ prev_byte = @dict_full.positive? ? @dict_buf.getbyte(prev_idx) : 0
1297
+
1298
+ # Safeguard: if prev_byte is nil, use 0
1299
+ prev_byte = 0 if prev_byte.nil?
1300
+
1301
+ # Combine current output position and prev_byte, then apply cached mask.
1302
+ # XZ Utils uses dict.pos (continuous write position) for this calculation.
1303
+ # We use @pos - LZ_DICT_INIT_POS (total bytes decoded) instead of @dict_full,
1304
+ # because @dict_full is clamped at @dict_size after the circular buffer wraps,
1305
+ # which would freeze the position-dependent literal model selection.
1306
+ output_pos = @pos - LZ_DICT_INIT_POS
1307
+ ((output_pos << 8) + prev_byte) & @literal_mask
2087
1308
  end
2088
1309
  end
2089
1310
  end