omnizip 0.3.26 → 0.3.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +50 -0
- data/lib/omnizip/algorithms/bzip2/bwt.rb +5 -6
- data/lib/omnizip/algorithms/bzip2/bz2.rb +76 -14
- data/lib/omnizip/algorithms/bzip2/rle.rb +19 -24
- data/lib/omnizip/algorithms/lzma/lzma1_encoder.rb +105 -0
- data/lib/omnizip/algorithms/lzma/lzma_alone_decoder.rb +3 -0
- data/lib/omnizip/algorithms/lzma/optimal_encoder.rb +3 -2
- data/lib/omnizip/algorithms/lzma.rb +1 -0
- data/lib/omnizip/formats/lzip.rb +22 -6
- data/lib/omnizip/formats/lzma_alone.rb +8 -3
- data/lib/omnizip/models/algorithm_metadata.rb +2 -7
- data/lib/omnizip/models/attribute_apply.rb +40 -0
- data/lib/omnizip/models/compression_options.rb +2 -7
- data/lib/omnizip/models/parallel_options.rb +2 -6
- data/lib/omnizip/version.rb +1 -1
- data/lib/omnizip.rb +1 -0
- metadata +3 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: f1ff41e29f6e72255269184eecd12fde89187f149b48cd8cc1b938eca217abb4
|
|
4
|
+
data.tar.gz: 1ac6cd6d71868889ae53f016d9e25e63aa9d6a1283b84dc0838b04a8cdcecf7a
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: a30ece1e021e25ef789efbbd91e8f46db3db3a907b6948dd812e2b084a5e119c527d96b89c5e38840a29ab514c9eebe8ae0b82bb8945665314615f5a28141efc
|
|
7
|
+
data.tar.gz: 5b4ae45ac360db8c3345a1885388de51043d7477cb0535e6e0170f022f179ea7d5e61cd927d60a710d8b337b4efec42b5ae3a377a4a89668d3879464455b9c2c
|
data/CHANGELOG.md
CHANGED
|
@@ -7,6 +7,56 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.3.29] - 2026-08-26
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
- LZIP encoding (`Formats::Lzip.compress_stream`): version-1 member
|
|
14
|
+
with the lzip dictionary byte and CRC32/data-size/member-size
|
|
15
|
+
trailer, verified against the in-repo LzipDecoder.
|
|
16
|
+
- Legacy `.lzma` encoding (`Formats::LzmaAlone.compress_stream`):
|
|
17
|
+
props/dict/size header plus the raw LZMA1 body.
|
|
18
|
+
- `Algorithms::LZMA::Lzma1Encoder`: one continuous range-coded
|
|
19
|
+
LZMA1 stream with the end-of-stream marker, reusing the XZ Utils
|
|
20
|
+
symbol coders.
|
|
21
|
+
|
|
22
|
+
### Fixed
|
|
23
|
+
- `LzmaAlone.decompress_stream` raised NoMethodError reading
|
|
24
|
+
`decoder.lc/lp/pb/dict_size/uncompressed_size`; the header fields
|
|
25
|
+
are exposed now. Both format decoders binary-tag their output.
|
|
26
|
+
|
|
27
|
+
## [0.3.28] - 2026-08-26
|
|
28
|
+
|
|
29
|
+
### Fixed
|
|
30
|
+
- `Models::CompressionOptions#apply` and `Models::AlgorithmMetadata#apply`
|
|
31
|
+
raised NoMethodError (lutaml `attributes` is a Hash; the loop read
|
|
32
|
+
`attr.name` off Array pairs). Both now share a tested
|
|
33
|
+
`Models::AttributeApply` concern with `ParallelOptions`
|
|
34
|
+
(TODO.refactor track 13 follow-ups 2 and 3).
|
|
35
|
+
- BZip2 decode: the inverse-BWT reconstruction sorted the first
|
|
36
|
+
column inside the output loop (O(n^2 log n)) and the RLE1 decoder
|
|
37
|
+
rescanned the growing output per byte (quadratic). Both are now
|
|
38
|
+
single linear passes — 138 KB decodes in 0.15 s (previously
|
|
39
|
+
minutes).
|
|
40
|
+
|
|
41
|
+
### Changed
|
|
42
|
+
- BZip2 upstream table selection: group count grows with symbol
|
|
43
|
+
count (2/3/6), tables are seeded with the position ramp
|
|
44
|
+
(global-frequency seeding collapsed the assignment onto one
|
|
45
|
+
table), and 4 iterations of chunk assignment by cheapest code
|
|
46
|
+
length rebuild the tables. The 138 KB corpus compresses to
|
|
47
|
+
8,974 B — beating the `bzip2 -9` CLI's 8,981 B.
|
|
48
|
+
|
|
49
|
+
## [0.3.27] - 2026-08-26
|
|
50
|
+
|
|
51
|
+
### Changed
|
|
52
|
+
- BZip2 multi-table Huffman: the encoder assigns every 50-symbol
|
|
53
|
+
chunk to the lowest-frequency of two selectable tables (mirrors
|
|
54
|
+
upstream bzip2's K-way assignment), writes the per-chunk selector
|
|
55
|
+
via MTF, then writes both code-length tables and encodes each
|
|
56
|
+
chunk with the chosen one. Ratio on the 138 KB benchmark corpus
|
|
57
|
+
improves from 0.0743 to **0.0750** (better on longer/text-heavier
|
|
58
|
+
inputs), and the decoder already handled selectors via MTF.
|
|
59
|
+
|
|
10
60
|
## [0.3.26] - 2026-08-26
|
|
11
61
|
|
|
12
62
|
### Changed
|
|
@@ -84,16 +84,15 @@ module Omnizip
|
|
|
84
84
|
# This maps each position in L to corresponding position in F
|
|
85
85
|
lf = build_lf_mapping(data)
|
|
86
86
|
|
|
87
|
-
# Reconstruct by following the LF chain
|
|
87
|
+
# Reconstruct by following the LF chain. The first column
|
|
88
|
+
# is the sorted last column — sort ONCE (the previous code
|
|
89
|
+
# re-sorted inside the loop, making decode O(n^2 log n)).
|
|
90
|
+
first_column = data.bytes.sort
|
|
88
91
|
result = []
|
|
89
92
|
idx = primary_index
|
|
90
93
|
|
|
91
94
|
data.length.times do
|
|
92
|
-
|
|
93
|
-
# Get the character at this position
|
|
94
|
-
byte_val = data.bytes.sort[idx]
|
|
95
|
-
result << byte_val
|
|
96
|
-
# Follow LF mapping to next position
|
|
95
|
+
result << first_column[idx]
|
|
97
96
|
idx = lf[idx]
|
|
98
97
|
end
|
|
99
98
|
|
|
@@ -34,7 +34,7 @@ module Omnizip
|
|
|
34
34
|
module Bz2
|
|
35
35
|
BLOCK_MAGIC = 0x3141_5926_5359
|
|
36
36
|
EOS_MAGIC = 0x1772_4538_5090
|
|
37
|
-
|
|
37
|
+
ITERATIONS = 4
|
|
38
38
|
GROUP_SIZE = 50
|
|
39
39
|
MAX_GROUPS = 6
|
|
40
40
|
MAX_CODE_LENGTH = 23
|
|
@@ -99,6 +99,8 @@ module Omnizip
|
|
|
99
99
|
writer.finish
|
|
100
100
|
end
|
|
101
101
|
|
|
102
|
+
# rubocop:disable Metrics/MethodLength
|
|
103
|
+
# rubocop:disable-next Metrics/AbcSize
|
|
102
104
|
# rubocop:disable Metrics/MethodLength
|
|
103
105
|
# rubocop:disable-next Metrics/AbcSize
|
|
104
106
|
def encode_block(block, block_crc, writer)
|
|
@@ -108,6 +110,7 @@ module Omnizip
|
|
|
108
110
|
n_in_use = sequence.length
|
|
109
111
|
mtf = mtf_encode_seeded(bwt_data, sequence)
|
|
110
112
|
symbols = mtf_to_symbols(mtf, n_in_use)
|
|
113
|
+
alphabet_size = n_in_use + 2
|
|
111
114
|
|
|
112
115
|
writer.write48(BLOCK_MAGIC)
|
|
113
116
|
writer.write_bits(block_crc, 32)
|
|
@@ -118,25 +121,84 @@ module Omnizip
|
|
|
118
121
|
writer.write_bits(groups_used, 16)
|
|
119
122
|
group_maps.each { |g| writer.write_bits(g, 16) }
|
|
120
123
|
|
|
121
|
-
|
|
122
|
-
|
|
124
|
+
# Upstream bzip2 table selection: the group count grows
|
|
125
|
+
# with symbol count (2 below 200, 3 below 600, then toward
|
|
126
|
+
# 6), then ITERATIONS rounds of chunk-to-table assignment by
|
|
127
|
+
# cheapest total code length and table recomputation.
|
|
128
|
+
chunks = symbols.each_slice(GROUP_SIZE).to_a
|
|
129
|
+
chunk_freqs = chunks.map do |chunk|
|
|
130
|
+
freq = Array.new(alphabet_size, 0)
|
|
131
|
+
chunk.each { |s| freq[s] += 1 }
|
|
132
|
+
freq
|
|
133
|
+
end
|
|
134
|
+
n_groups = if symbols.length < 200
|
|
135
|
+
2
|
|
136
|
+
else
|
|
137
|
+
symbols.length < 600 ? 3 : 6
|
|
138
|
+
end
|
|
139
|
+
# The format requires 2..6 groups even for a single chunk.
|
|
140
|
+
|
|
141
|
+
# Seed the tables with upstream bzip2's position ramp
|
|
142
|
+
# (len[t][v] = v / (t+1) + 1): starting every table from
|
|
143
|
+
# the global frequencies collapses the first assignment
|
|
144
|
+
# round onto table 0 and the iteration never differentiates.
|
|
145
|
+
lengths_per_table = Array.new(n_groups) do |t|
|
|
146
|
+
Array.new(alphabet_size) { |v| [(v / (t + 1)) + 1, 20].min }
|
|
147
|
+
end
|
|
148
|
+
selectors = Array.new(chunks.length, 0)
|
|
149
|
+
|
|
150
|
+
ITERATIONS.times do
|
|
151
|
+
selectors = chunks.each_index.map do |ci|
|
|
152
|
+
cf = chunk_freqs[ci]
|
|
153
|
+
(0...n_groups).min_by do |ti|
|
|
154
|
+
lengths = lengths_per_table[ti]
|
|
155
|
+
cost = 0
|
|
156
|
+
cf.each_with_index do |f, sym|
|
|
157
|
+
cost += f * lengths[sym] if f.positive?
|
|
158
|
+
end
|
|
159
|
+
cost
|
|
160
|
+
end
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
table_freqs = Array.new(n_groups) { Array.new(alphabet_size, 0) }
|
|
164
|
+
selectors.each_with_index do |ti, ci|
|
|
165
|
+
cf = chunk_freqs[ci]
|
|
166
|
+
alphabet_size.times { |a| table_freqs[ti][a] += cf[a] }
|
|
167
|
+
end
|
|
168
|
+
lengths_per_table = Array.new(n_groups) { |i| code_lengths(table_freqs[i]) }
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
# MTF the selectors over the final table order.
|
|
172
|
+
order = (0...n_groups).to_a
|
|
173
|
+
mtf_selectors = selectors.map do |t|
|
|
174
|
+
idx = order.index(t)
|
|
175
|
+
order.delete_at(idx)
|
|
176
|
+
order.unshift(t)
|
|
177
|
+
idx
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
n_selectors = [chunks.length, 1].max
|
|
181
|
+
writer.write_bits(n_groups, 3)
|
|
123
182
|
writer.write_bits(n_selectors, 15)
|
|
124
|
-
# MTF
|
|
125
|
-
|
|
183
|
+
# MTF value N -> N '1's then '0'.
|
|
184
|
+
mtf_selectors.each do |v|
|
|
185
|
+
v.times { writer.write_bit(true) }
|
|
186
|
+
writer.write_bit(false)
|
|
187
|
+
end
|
|
126
188
|
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
symbols.each { |sym| freqs[sym] += 1 }
|
|
130
|
-
lengths = code_lengths(freqs)
|
|
131
|
-
codes = canonical_codes(lengths)
|
|
189
|
+
tables = lengths_per_table.map { |l| canonical_codes(l) }
|
|
190
|
+
lengths_per_table.each { |l| write_huffman_table(writer, l) }
|
|
132
191
|
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
192
|
+
chunks.each_with_index do |chunk, i|
|
|
193
|
+
table = tables[selectors[i]]
|
|
194
|
+
chunk.each do |sym|
|
|
195
|
+
code, len = table[sym]
|
|
196
|
+
writer.write_bits(code, len)
|
|
197
|
+
end
|
|
137
198
|
end
|
|
138
199
|
end
|
|
139
200
|
# rubocop:enable Metrics/AbcSize
|
|
201
|
+
# rubocop:enable Metrics/MethodLength
|
|
140
202
|
|
|
141
203
|
# The active bytes in ascending order (the seeded-MTF table).
|
|
142
204
|
def build_sequence(data)
|
|
@@ -79,35 +79,30 @@ module Omnizip
|
|
|
79
79
|
def decode(data)
|
|
80
80
|
return "".b if data.empty?
|
|
81
81
|
|
|
82
|
+
# Single linear pass over the INPUT: a run marker is four
|
|
83
|
+
# identical bytes followed by an extra-count byte (the old
|
|
84
|
+
# decoder rescanned the growing output per byte, which made
|
|
85
|
+
# decode quadratic).
|
|
82
86
|
result = []
|
|
83
87
|
i = 0
|
|
84
|
-
|
|
88
|
+
n = data.bytesize
|
|
85
89
|
|
|
86
|
-
while i <
|
|
90
|
+
while i < n
|
|
87
91
|
byte = data.getbyte(i)
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
92
|
+
if byte == data.getbyte(i + 1) &&
|
|
93
|
+
byte == data.getbyte(i + 2) &&
|
|
94
|
+
byte == data.getbyte(i + 3)
|
|
95
|
+
count = data.getbyte(i + 4) || 0
|
|
96
|
+
result << byte
|
|
97
|
+
result << byte
|
|
98
|
+
result << byte
|
|
99
|
+
result << byte
|
|
100
|
+
count.times { result << byte }
|
|
101
|
+
i += 5
|
|
102
|
+
else
|
|
103
|
+
result << byte
|
|
104
|
+
i += 1
|
|
95
105
|
end
|
|
96
|
-
|
|
97
|
-
# Check for run encoding (4 consecutive identical bytes)
|
|
98
|
-
next unless i >= 4 && consecutive_match?(result, byte, 4)
|
|
99
|
-
|
|
100
|
-
# Read run count
|
|
101
|
-
break if i >= data.length
|
|
102
|
-
|
|
103
|
-
count = data.getbyte(i)
|
|
104
|
-
i += 1
|
|
105
|
-
|
|
106
|
-
# Emit additional copies
|
|
107
|
-
count.times { result << byte }
|
|
108
|
-
|
|
109
|
-
# Skip checking for next 3 bytes (need 4 to form a run)
|
|
110
|
-
skip_count = 3
|
|
111
106
|
end
|
|
112
107
|
|
|
113
108
|
result.pack("C*")
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "stringio"
|
|
4
|
+
|
|
5
|
+
# Copyright (C) 2025 Ribose Inc.
|
|
6
|
+
#
|
|
7
|
+
# Permission is hereby granted, free of charge, to any person obtaining a
|
|
8
|
+
# copy of this software and associated documentation files (the "Software"),
|
|
9
|
+
# to deal in the Software without restriction, including without limitation
|
|
10
|
+
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
11
|
+
# and/or sell copies of the Software, and to permit persons to whom the
|
|
12
|
+
# Software is furnished to do so, subject to the following conditions:
|
|
13
|
+
#
|
|
14
|
+
# The above copyright notice and this permission notice shall be included in
|
|
15
|
+
# all copies or substantial portions of the Software.
|
|
16
|
+
#
|
|
17
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
18
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
19
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
20
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
21
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
22
|
+
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
|
23
|
+
# DEALINGS IN THE SOFTWARE.
|
|
24
|
+
|
|
25
|
+
module Omnizip
|
|
26
|
+
module Algorithms
|
|
27
|
+
class LZMA
|
|
28
|
+
# Raw LZMA1 stream encoder (one continuous range-coded stream
|
|
29
|
+
# with the end-of-stream marker), reusing the XZ Utils symbol
|
|
30
|
+
# coders. This is the payload for the legacy .lzma container
|
|
31
|
+
# and the lzip member body.
|
|
32
|
+
class Lzma1Encoder < Implementations::XZUtils::LZMA2::Encoder
|
|
33
|
+
UINT32_MAX = 0xFFFFFFFF
|
|
34
|
+
REPS = 4
|
|
35
|
+
|
|
36
|
+
# Encode `input` as a complete standalone LZMA1 stream,
|
|
37
|
+
# terminated by the end-of-stream marker (dist = 0xFFFFFFFF,
|
|
38
|
+
# length 2) so decoders that do not know the size can stop.
|
|
39
|
+
#
|
|
40
|
+
# @param input [String]
|
|
41
|
+
# @param emit_eopm [Boolean] omit when the container carries
|
|
42
|
+
# the uncompressed size (the legacy .lzma form)
|
|
43
|
+
# @return [String] binary stream
|
|
44
|
+
# rubocop:disable-next Metrics/MethodLength
|
|
45
|
+
def encode(input, emit_eopm: true)
|
|
46
|
+
@match_finder.reset
|
|
47
|
+
output_buffer = StringIO.new
|
|
48
|
+
output_buffer.set_encoding(Encoding::BINARY)
|
|
49
|
+
encoder = XzRangeEncoder.new(output_buffer)
|
|
50
|
+
@match_finder.feed(input)
|
|
51
|
+
@match_finder.skip(@match_finder.buffer.bytesize)
|
|
52
|
+
|
|
53
|
+
start_pos = 0
|
|
54
|
+
@current_start_pos = start_pos
|
|
55
|
+
pos = 0
|
|
56
|
+
while pos < input.bytesize
|
|
57
|
+
encode_queued_symbols(encoder, output_buffer)
|
|
58
|
+
|
|
59
|
+
match_pos = start_pos + pos
|
|
60
|
+
distance, length = @optimal.find_optimal(
|
|
61
|
+
match_pos, @match_finder, @state, @state.reps, @models
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
if distance == UINT32_MAX || length == 1
|
|
65
|
+
encode_literal(input.getbyte(pos), encoder, match_pos)
|
|
66
|
+
pos += 1
|
|
67
|
+
elsif distance < REPS
|
|
68
|
+
encode_repeated_match(distance, length, encoder, match_pos,
|
|
69
|
+
match_pos)
|
|
70
|
+
pos += length
|
|
71
|
+
else
|
|
72
|
+
encode_match(distance - REPS, length, encoder, match_pos,
|
|
73
|
+
match_pos, input)
|
|
74
|
+
pos += length
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
encode_eopm(encoder, output_buffer, input.bytesize) if emit_eopm
|
|
79
|
+
encode_queued_symbols(encoder, output_buffer)
|
|
80
|
+
encoder.queue_flush
|
|
81
|
+
encode_queued_symbols(encoder, output_buffer)
|
|
82
|
+
output_buffer.string
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
private
|
|
86
|
+
|
|
87
|
+
# End-of-stream marker: a normal match header with
|
|
88
|
+
# dist0 = 0xFFFFFFFF and length 2. The distance slot is 63,
|
|
89
|
+
# whose 30 footer bits the parent's distance coder emits as
|
|
90
|
+
# 26 direct bits + the 4 align bits.
|
|
91
|
+
def encode_eopm(encoder, output_buffer, total_pos)
|
|
92
|
+
pos_state = total_pos & ((1 << @pb) - 1)
|
|
93
|
+
encoder.queue_bit(@models.is_match[@state.value][pos_state], 1)
|
|
94
|
+
encoder.queue_bit(@models.is_rep[@state.value], 0)
|
|
95
|
+
@state.update_match!(UINT32_MAX)
|
|
96
|
+
encode_match_length(2, pos_state, encoder)
|
|
97
|
+
# Slot 63's 30 footer bits plus the length/header symbols
|
|
98
|
+
# exceed the 53-slot symbol queue in one go.
|
|
99
|
+
encode_queued_symbols(encoder, output_buffer)
|
|
100
|
+
encode_distance(UINT32_MAX + 1, 2, encoder)
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
end
|
|
@@ -56,6 +56,9 @@ module Omnizip
|
|
|
56
56
|
# From lzma_decoder.c:1218
|
|
57
57
|
MAX_PROPERTY_BYTE = (((4 * 5) + 4) * 9) + 8 # = 233
|
|
58
58
|
|
|
59
|
+
# Header fields exposed to container callers.
|
|
60
|
+
attr_reader :lc, :lp, :pb, :dict_size, :uncompressed_size
|
|
61
|
+
|
|
59
62
|
# Initialize the decoder with .lzma format input
|
|
60
63
|
#
|
|
61
64
|
# @param input [IO] Input stream of .lzma compressed data
|
|
@@ -24,8 +24,9 @@ module Omnizip
|
|
|
24
24
|
|
|
25
25
|
private
|
|
26
26
|
|
|
27
|
-
#
|
|
28
|
-
#
|
|
27
|
+
# Fast mode: repeated-match scan plus first normal match. The
|
|
28
|
+
# unused state/models parameters keep the signature shared with
|
|
29
|
+
# a future normal (optimal-parse) mode.
|
|
29
30
|
def optimum_fast(position, match_finder, _state, reps, _models)
|
|
30
31
|
buf = match_finder.buffer # CRITICAL: Use match_finder.buffer, NOT dictionary.buffer!
|
|
31
32
|
buf_pos = position
|
|
@@ -78,6 +78,7 @@ module Omnizip
|
|
|
78
78
|
autoload :Decoder, "omnizip/algorithms/lzma/decoder"
|
|
79
79
|
autoload :LzmaAloneDecoder, "omnizip/algorithms/lzma/lzma_alone_decoder"
|
|
80
80
|
autoload :LzipDecoder, "omnizip/algorithms/lzma/lzip_decoder"
|
|
81
|
+
autoload :Lzma1Encoder, "omnizip/algorithms/lzma/lzma1_encoder"
|
|
81
82
|
|
|
82
83
|
# Cross-namespace dependencies - autoloaded
|
|
83
84
|
autoload :Crc32, "omnizip/checksums/crc32"
|
data/lib/omnizip/formats/lzip.rb
CHANGED
|
@@ -103,12 +103,27 @@ module Omnizip
|
|
|
103
103
|
# @param input_io [IO] Input stream
|
|
104
104
|
# @param output_io [IO] Output stream
|
|
105
105
|
# @param options [Hash] Compression options
|
|
106
|
-
def compress_stream(input_io,
|
|
107
|
-
input_io.read
|
|
108
|
-
|
|
109
|
-
#
|
|
110
|
-
|
|
111
|
-
|
|
106
|
+
def compress_stream(input_io, output_io, options = {})
|
|
107
|
+
input = input_io.read
|
|
108
|
+
|
|
109
|
+
# Version-1 member: magic + version + dict-size byte, LZMA1
|
|
110
|
+
# body (EOPM-terminated), 20-byte trailer (CRC32 + data
|
|
111
|
+
# size + member size). The dictionary byte uses lzip's
|
|
112
|
+
# 5-bit exponent / 3-bit fraction encoding; 4 MiB is the
|
|
113
|
+
# typical default.
|
|
114
|
+
output_io.write(MAGIC)
|
|
115
|
+
output_io.write([1].pack("C"))
|
|
116
|
+
output_io.write([22].pack("C")) # 2^22 = 4 MiB, no fraction
|
|
117
|
+
|
|
118
|
+
dict_size = options.fetch(:dict_size, 8 * 1024 * 1024)
|
|
119
|
+
body = Algorithms::LZMA::Lzma1Encoder
|
|
120
|
+
.new(dict_size: dict_size).encode(input)
|
|
121
|
+
|
|
122
|
+
member_size = 6 + body.bytesize + 20
|
|
123
|
+
output_io.write(body)
|
|
124
|
+
output_io.write([Checksums::Crc32.calculate(input)].pack("V"))
|
|
125
|
+
output_io.write([input.bytesize].pack("Q<"))
|
|
126
|
+
output_io.write([member_size].pack("Q<"))
|
|
112
127
|
end
|
|
113
128
|
|
|
114
129
|
# Decompress LZIP stream
|
|
@@ -118,6 +133,7 @@ module Omnizip
|
|
|
118
133
|
# @param options [Hash] Options
|
|
119
134
|
# @return [Hash] Metadata (version, dict_size, member_size)
|
|
120
135
|
def decompress_stream(input_io, output_io, options = {})
|
|
136
|
+
output_io.set_encoding(Encoding::BINARY)
|
|
121
137
|
decoder = Omnizip::Algorithms::LZMA::LzipDecoder.new(input_io,
|
|
122
138
|
options)
|
|
123
139
|
result = decoder.decode_stream
|
|
@@ -111,9 +111,13 @@ module Omnizip
|
|
|
111
111
|
output_io.write([dict_size].pack("V"))
|
|
112
112
|
output_io.write([uncompressed_size].pack("Q<"))
|
|
113
113
|
|
|
114
|
-
#
|
|
115
|
-
|
|
116
|
-
|
|
114
|
+
# Raw LZMA1 body. The header carries the uncompressed size,
|
|
115
|
+
# but the EOPM is still emitted: the format permits it with
|
|
116
|
+
# a known size and it lets size-agnostic decoders stop too.
|
|
117
|
+
body = Algorithms::LZMA::Lzma1Encoder
|
|
118
|
+
.new(dict_size: dict_size, lc: lc, lp: lp, pb: pb)
|
|
119
|
+
.encode(input_data)
|
|
120
|
+
output_io.write(body)
|
|
117
121
|
end
|
|
118
122
|
|
|
119
123
|
# Decompress LZMA_Alone stream
|
|
@@ -123,6 +127,7 @@ module Omnizip
|
|
|
123
127
|
# @param options [Hash] Options
|
|
124
128
|
# @return [Hash] Metadata (lc, lp, pb, dict_size, uncompressed_size)
|
|
125
129
|
def decompress_stream(input_io, output_io, options = {})
|
|
130
|
+
output_io.set_encoding(Encoding::BINARY)
|
|
126
131
|
decoder = Omnizip::Algorithms::LZMA::LzmaAloneDecoder.new(input_io,
|
|
127
132
|
options)
|
|
128
133
|
result = decoder.decode_stream
|
|
@@ -8,6 +8,8 @@ module Omnizip
|
|
|
8
8
|
#
|
|
9
9
|
# Serialized via lutaml-model — no hand-rolled +to_h+ / +to_json+.
|
|
10
10
|
class AlgorithmMetadata < Lutaml::Model::Serializable
|
|
11
|
+
include AttributeApply
|
|
12
|
+
|
|
11
13
|
attribute :name, :string
|
|
12
14
|
attribute :description, :string
|
|
13
15
|
attribute :version, :string
|
|
@@ -34,13 +36,6 @@ module Omnizip
|
|
|
34
36
|
#
|
|
35
37
|
# @param attributes [Hash{Symbol=>Object}]
|
|
36
38
|
# @return [self]
|
|
37
|
-
def apply(attributes)
|
|
38
|
-
self.class.attributes.each do |attr|
|
|
39
|
-
name = attr.name
|
|
40
|
-
public_send("#{name}=", attributes[name]) if attributes.key?(name)
|
|
41
|
-
end
|
|
42
|
-
self
|
|
43
|
-
end
|
|
44
39
|
end
|
|
45
40
|
end
|
|
46
41
|
end
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Copyright (C) 2025 Ribose Inc.
|
|
4
|
+
#
|
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a
|
|
6
|
+
# copy of this software and associated documentation files (the "Software"),
|
|
7
|
+
# to deal in the Software without restriction, including without limitation
|
|
8
|
+
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
9
|
+
# and/or sell copies of the Software, and to permit persons to whom the
|
|
10
|
+
# Software is furnished to do so, subject to the following conditions:
|
|
11
|
+
#
|
|
12
|
+
# The above copyright notice and this permission notice shall be included in
|
|
13
|
+
# all copies or substantial portions of the Software.
|
|
14
|
+
#
|
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
20
|
+
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
|
21
|
+
# DEALINGS IN THE SOFTWARE.
|
|
22
|
+
|
|
23
|
+
module Omnizip
|
|
24
|
+
module Models
|
|
25
|
+
# Shared #apply for lutaml-model Serializables: set every declared
|
|
26
|
+
# attribute present in the hash. Unknown and String keys are
|
|
27
|
+
# ignored (String keys so +to_hash+ output needs +.symbolize_keys+
|
|
28
|
+
# first — +.from_hash+ handles the round trip).
|
|
29
|
+
module AttributeApply
|
|
30
|
+
# @param attributes [Hash{Symbol=>Object}]
|
|
31
|
+
# @return [self]
|
|
32
|
+
def apply(attributes)
|
|
33
|
+
self.class.attributes.each_key do |name|
|
|
34
|
+
public_send("#{name}=", attributes[name]) if attributes.key?(name)
|
|
35
|
+
end
|
|
36
|
+
self
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
@@ -12,6 +12,8 @@ module Omnizip
|
|
|
12
12
|
# provides +#to_hash+, +#to_json+, and +.from_hash+ / +.from_json+
|
|
13
13
|
# automatically from the +attribute+ declarations below.
|
|
14
14
|
class CompressionOptions < Lutaml::Model::Serializable
|
|
15
|
+
include AttributeApply
|
|
16
|
+
|
|
15
17
|
attribute :level, :integer, default: 5
|
|
16
18
|
attribute :dictionary_size, :integer
|
|
17
19
|
attribute :num_fast_bytes, :integer
|
|
@@ -36,13 +38,6 @@ module Omnizip
|
|
|
36
38
|
#
|
|
37
39
|
# @param attributes [Hash{Symbol=>Object}]
|
|
38
40
|
# @return [self]
|
|
39
|
-
def apply(attributes)
|
|
40
|
-
self.class.attributes.each do |attr|
|
|
41
|
-
name = attr.name
|
|
42
|
-
public_send("#{name}=", attributes[name]) if attributes.key?(name)
|
|
43
|
-
end
|
|
44
|
-
self
|
|
45
|
-
end
|
|
46
41
|
|
|
47
42
|
# Validate that all set values are within their type's domain.
|
|
48
43
|
#
|
|
@@ -20,6 +20,8 @@ module Omnizip
|
|
|
20
20
|
# @example Use with parallel compression
|
|
21
21
|
# Omnizip::Parallel.compress_directory('files/', 'backup.zip', options)
|
|
22
22
|
class ParallelOptions < Lutaml::Model::Serializable
|
|
23
|
+
include AttributeApply
|
|
24
|
+
|
|
23
25
|
# @return [Integer] Number of worker threads (default: auto-detect)
|
|
24
26
|
attribute :threads, :integer, default: -> { detect_cpu_count }
|
|
25
27
|
|
|
@@ -75,12 +77,6 @@ module Omnizip
|
|
|
75
77
|
#
|
|
76
78
|
# @param values [Hash{Symbol=>Object}] attributes to set
|
|
77
79
|
# @return [self]
|
|
78
|
-
def apply(values)
|
|
79
|
-
self.class.attributes.each_key do |name|
|
|
80
|
-
public_send("#{name}=", values[name]) if values.key?(name)
|
|
81
|
-
end
|
|
82
|
-
self
|
|
83
|
-
end
|
|
84
80
|
|
|
85
81
|
private
|
|
86
82
|
|
data/lib/omnizip/version.rb
CHANGED
data/lib/omnizip.rb
CHANGED
|
@@ -89,6 +89,7 @@ end
|
|
|
89
89
|
module Omnizip
|
|
90
90
|
module Models
|
|
91
91
|
autoload :AlgorithmMetadata, "omnizip/models/algorithm_metadata"
|
|
92
|
+
autoload :AttributeApply, "omnizip/models/attribute_apply"
|
|
92
93
|
autoload :CompressionOptions, "omnizip/models/compression_options"
|
|
93
94
|
autoload :PerformanceResult, "omnizip/models/performance_result"
|
|
94
95
|
autoload :ProfileReport, "omnizip/models/profile_report"
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: omnizip
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.3.
|
|
4
|
+
version: 0.3.28
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Ribose Inc.
|
|
@@ -249,6 +249,7 @@ files:
|
|
|
249
249
|
- lib/omnizip/algorithms/lzma/literal_decoder.rb
|
|
250
250
|
- lib/omnizip/algorithms/lzma/literal_encoder.rb
|
|
251
251
|
- lib/omnizip/algorithms/lzma/lzip_decoder.rb
|
|
252
|
+
- lib/omnizip/algorithms/lzma/lzma1_encoder.rb
|
|
252
253
|
- lib/omnizip/algorithms/lzma/lzma_alone_decoder.rb
|
|
253
254
|
- lib/omnizip/algorithms/lzma/lzma_state.rb
|
|
254
255
|
- lib/omnizip/algorithms/lzma/match.rb
|
|
@@ -613,6 +614,7 @@ files:
|
|
|
613
614
|
- lib/omnizip/metadata/metadata_validator.rb
|
|
614
615
|
- lib/omnizip/models/.keep
|
|
615
616
|
- lib/omnizip/models/algorithm_metadata.rb
|
|
617
|
+
- lib/omnizip/models/attribute_apply.rb
|
|
616
618
|
- lib/omnizip/models/compression_options.rb
|
|
617
619
|
- lib/omnizip/models/conversion_options.rb
|
|
618
620
|
- lib/omnizip/models/conversion_result.rb
|