zip_tricks 2.6.0 → 2.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/zip_tricks/block_deflate.rb +4 -4
- data/lib/zip_tricks/block_write.rb +4 -4
- data/lib/zip_tricks/manifest.rb +13 -13
- data/lib/zip_tricks/rack_body.rb +4 -4
- data/lib/zip_tricks/remote_io.rb +13 -12
- data/lib/zip_tricks/remote_uncap.rb +12 -12
- data/lib/zip_tricks/stored_size_estimator.rb +3 -3
- data/lib/zip_tricks/stream_crc32.rb +4 -4
- data/lib/zip_tricks/streamer.rb +18 -18
- data/lib/zip_tricks/write_and_tell.rb +5 -5
- data/lib/zip_tricks.rb +1 -1
- data/spec/zip_tricks/block_deflate_spec.rb +16 -16
- data/spec/zip_tricks/block_write_spec.rb +16 -16
- data/spec/zip_tricks/manifest_spec.rb +11 -11
- data/spec/zip_tricks/rack_body_spec.rb +6 -6
- data/spec/zip_tricks/remote_io_spec.rb +22 -14
- data/spec/zip_tricks/remote_uncap_spec.rb +20 -20
- data/spec/zip_tricks/stored_size_estimator_spec.rb +4 -4
- data/spec/zip_tricks/stream_crc32_spec.rb +7 -7
- data/spec/zip_tricks/streamer_spec.rb +47 -47
- data/spec/zip_tricks/write_and_tell_spec.rb +6 -6
- data/zip_tricks.gemspec +3 -3
- metadata +2 -2
@@ -6,44 +6,44 @@ describe ZipTricks::BlockWrite do
|
|
6
6
|
adapter = described_class.new{|s|
|
7
7
|
blobs << s
|
8
8
|
}
|
9
|
-
|
9
|
+
|
10
10
|
adapter << 'hello'
|
11
11
|
adapter << 'world'
|
12
12
|
adapter << '!'
|
13
|
-
|
13
|
+
|
14
14
|
expect(blobs).to eq(['hello', 'world', '!'])
|
15
15
|
end
|
16
|
-
|
16
|
+
|
17
17
|
it 'supports chained shovel' do
|
18
18
|
blobs = []
|
19
19
|
adapter = described_class.new{|s|
|
20
20
|
blobs << s
|
21
21
|
}
|
22
|
-
|
22
|
+
|
23
23
|
adapter << 'hello' << 'world' << '!'
|
24
|
-
|
24
|
+
|
25
25
|
expect(blobs).to eq(['hello', 'world', '!'])
|
26
26
|
end
|
27
|
-
|
27
|
+
|
28
28
|
it 'can write in all possible encodings, even if the strings are frozen' do
|
29
29
|
destination = ''.encode(Encoding::BINARY)
|
30
|
-
|
30
|
+
|
31
31
|
accum_string = ''
|
32
32
|
adapter = described_class.new{|s| accum_string << s }
|
33
|
-
|
33
|
+
|
34
34
|
adapter << 'hello'
|
35
35
|
adapter << 'привет'
|
36
36
|
adapter << 'привет'.freeze
|
37
37
|
adapter << '!'
|
38
38
|
adapter << SecureRandom.random_bytes(1024)
|
39
|
-
|
39
|
+
|
40
40
|
expect(accum_string.bytesize).to eq(1054)
|
41
41
|
end
|
42
|
-
|
42
|
+
|
43
43
|
it 'can be closed' do
|
44
44
|
expect(described_class.new{}.close).to be_nil
|
45
45
|
end
|
46
|
-
|
46
|
+
|
47
47
|
it 'forces the written strings to binary encoding' do
|
48
48
|
blobs = []
|
49
49
|
adapter = described_class.new{|s|
|
@@ -55,7 +55,7 @@ describe ZipTricks::BlockWrite do
|
|
55
55
|
expect(blobs).not_to be_empty
|
56
56
|
blobs.each {|s| expect(s.encoding).to eq(Encoding::BINARY) }
|
57
57
|
end
|
58
|
-
|
58
|
+
|
59
59
|
it 'omits strings of zero length' do
|
60
60
|
blobs = []
|
61
61
|
adapter = described_class.new{|s|
|
@@ -66,7 +66,7 @@ describe ZipTricks::BlockWrite do
|
|
66
66
|
adapter << '!'
|
67
67
|
expect(blobs).to eq(['hello', '!'])
|
68
68
|
end
|
69
|
-
|
69
|
+
|
70
70
|
it 'omits nils' do
|
71
71
|
blobs = []
|
72
72
|
adapter = described_class.new{|s|
|
@@ -77,17 +77,17 @@ describe ZipTricks::BlockWrite do
|
|
77
77
|
adapter << '!'
|
78
78
|
expect(blobs).to eq(['hello', '!'])
|
79
79
|
end
|
80
|
-
|
80
|
+
|
81
81
|
it 'raises a TypeError on specific unsupported methods' do
|
82
82
|
adapter = described_class.new {|s| }
|
83
83
|
expect {
|
84
84
|
adapter.seek(123)
|
85
85
|
}.to raise_error(/non\-rewindable/)
|
86
|
-
|
86
|
+
|
87
87
|
expect {
|
88
88
|
adapter.to_s
|
89
89
|
}.to raise_error(/non\-rewindable/)
|
90
|
-
|
90
|
+
|
91
91
|
expect {
|
92
92
|
adapter.pos = 123
|
93
93
|
}.to raise_error(/non\-rewindable/)
|
@@ -6,53 +6,53 @@ describe ZipTricks::Manifest do
|
|
6
6
|
raw_file_1 = SecureRandom.random_bytes(1024 * 20)
|
7
7
|
raw_file_2 = SecureRandom.random_bytes(1024 * 128)
|
8
8
|
raw_file_3 = SecureRandom.random_bytes(1258695)
|
9
|
-
|
9
|
+
|
10
10
|
manifest, bytesize = described_class.build do | builder |
|
11
11
|
r = builder.add_stored_entry(name: "first-file.bin", size_uncompressed: raw_file_1.size)
|
12
12
|
expect(r).to eq(builder), "add_stored_entry should return self"
|
13
|
-
|
13
|
+
|
14
14
|
builder.add_stored_entry(name: "second-file.bin", size_uncompressed: raw_file_2.size)
|
15
|
-
|
15
|
+
|
16
16
|
r = builder.add_compressed_entry(name: "second-file-comp.bin", size_uncompressed: raw_file_2.size,
|
17
17
|
size_compressed: raw_file_3.size, segment_info: 'http://example.com/second-file-deflated-segment.bin')
|
18
18
|
expect(r).to eq(builder), "add_compressed_entry should return self"
|
19
19
|
end
|
20
|
-
|
20
|
+
|
21
21
|
require 'range_utils'
|
22
|
-
|
22
|
+
|
23
23
|
expect(manifest).to be_kind_of(Array)
|
24
24
|
total_size_of_all_parts = manifest.inject(0) do | total_bytes, span |
|
25
25
|
total_bytes + RangeUtils.size_from_range(span.byte_range_in_zip)
|
26
26
|
end
|
27
27
|
expect(total_size_of_all_parts).to eq(1410595)
|
28
28
|
expect(bytesize).to eq(1410595)
|
29
|
-
|
29
|
+
|
30
30
|
expect(manifest.length).to eq(7)
|
31
|
-
|
31
|
+
|
32
32
|
first_header = manifest[0]
|
33
33
|
expect(first_header.part_type).to eq(:entry_header)
|
34
34
|
expect(first_header.byte_range_in_zip).to eq(0..43)
|
35
35
|
expect(first_header.filename).to eq("first-file.bin")
|
36
36
|
expect(first_header.additional_metadata).to be_nil
|
37
|
-
|
37
|
+
|
38
38
|
first_body = manifest[1]
|
39
39
|
expect(first_body.part_type).to eq(:entry_body)
|
40
40
|
expect(first_body.byte_range_in_zip).to eq(44..20523)
|
41
41
|
expect(first_body.filename).to eq("first-file.bin")
|
42
42
|
expect(first_body.additional_metadata).to be_nil
|
43
|
-
|
43
|
+
|
44
44
|
third_header = manifest[4]
|
45
45
|
expect(third_header.part_type).to eq(:entry_header)
|
46
46
|
expect(third_header.byte_range_in_zip).to eq(151641..151690)
|
47
47
|
expect(third_header.filename).to eq("second-file-comp.bin")
|
48
48
|
expect(third_header.additional_metadata).to eq("http://example.com/second-file-deflated-segment.bin")
|
49
|
-
|
49
|
+
|
50
50
|
third_body = manifest[5]
|
51
51
|
expect(third_body.part_type).to eq(:entry_body)
|
52
52
|
expect(third_body.byte_range_in_zip).to eq(151691..1410385)
|
53
53
|
expect(third_body.filename).to eq("second-file-comp.bin")
|
54
54
|
expect(third_body.additional_metadata).to eq("http://example.com/second-file-deflated-segment.bin")
|
55
|
-
|
55
|
+
|
56
56
|
cd = manifest[-1]
|
57
57
|
expect(cd.part_type).to eq(:central_directory)
|
58
58
|
expect(cd.byte_range_in_zip).to eq(1410386..1410594)
|
@@ -3,22 +3,22 @@ require_relative '../spec_helper'
|
|
3
3
|
describe ZipTricks::RackBody do
|
4
4
|
it 'is usable as a Rack response body, supports each() and close()' do
|
5
5
|
output_buf = Tempfile.new('output')
|
6
|
-
|
6
|
+
|
7
7
|
file_body = SecureRandom.random_bytes(1024 * 1024 + 8981)
|
8
|
-
|
8
|
+
|
9
9
|
body = described_class.new do | zip |
|
10
10
|
zip.add_stored_entry("A file", file_body.bytesize, Zlib.crc32(file_body))
|
11
11
|
zip << file_body
|
12
12
|
end
|
13
|
-
|
13
|
+
|
14
14
|
body.each do | some_data |
|
15
15
|
output_buf << some_data
|
16
16
|
end
|
17
17
|
body.close
|
18
|
-
|
18
|
+
|
19
19
|
output_buf.rewind
|
20
20
|
expect(output_buf.size).to eq(1057667)
|
21
|
-
|
21
|
+
|
22
22
|
per_filename = {}
|
23
23
|
Zip::File.open(output_buf.path) do |zip_file|
|
24
24
|
# Handle entries one by one
|
@@ -27,7 +27,7 @@ describe ZipTricks::RackBody do
|
|
27
27
|
per_filename[entry.name] = entry.get_input_stream.read
|
28
28
|
end
|
29
29
|
end
|
30
|
-
|
30
|
+
|
31
31
|
expect(per_filename).to have_key('A file')
|
32
32
|
expect(per_filename['A file'].bytesize).to eq(file_body.bytesize)
|
33
33
|
end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
3
|
describe ZipTricks::RemoteIO do
|
4
|
-
|
4
|
+
|
5
5
|
context 'working with the fetcher object' do
|
6
6
|
it 'asks the fetcher object to obtain the object size and the actual data when reading' do
|
7
7
|
mock_fetcher = double(request_object_size: 120, request_range: 'abc')
|
@@ -9,17 +9,17 @@ describe ZipTricks::RemoteIO do
|
|
9
9
|
expect(subject.read(3)).to eq('abc')
|
10
10
|
end
|
11
11
|
end
|
12
|
-
|
12
|
+
|
13
13
|
context 'when it internally addresses a remote resource' do
|
14
14
|
it 'requests the size of the resource once via #request_object_size and does neet to read if resource is empty' do
|
15
15
|
subject = described_class.new
|
16
16
|
expect(subject).to receive(:request_object_size).and_return(0)
|
17
17
|
expect(subject.read).to be_nil
|
18
18
|
end
|
19
|
-
|
19
|
+
|
20
20
|
it 'performs remote reads when repeatedly requesting the same chunk, via #request_range' do
|
21
21
|
subject = described_class.new
|
22
|
-
|
22
|
+
|
23
23
|
expect(subject).to receive(:request_object_size).and_return(120)
|
24
24
|
allow(subject).to receive(:request_range) {|range|
|
25
25
|
expect(range).to eq(5..14)
|
@@ -31,7 +31,7 @@ describe ZipTricks::RemoteIO do
|
|
31
31
|
end
|
32
32
|
end
|
33
33
|
end
|
34
|
-
|
34
|
+
|
35
35
|
describe '#seek' do
|
36
36
|
context 'with an unsupported mode' do
|
37
37
|
it 'raises an error' do
|
@@ -55,7 +55,7 @@ describe ZipTricks::RemoteIO do
|
|
55
55
|
it 'seens to 10 bytes to the end of the IO' do
|
56
56
|
uncap = described_class.new
|
57
57
|
expect(uncap).to receive(:request_object_size).and_return(100)
|
58
|
-
|
58
|
+
|
59
59
|
mode = IO::SEEK_END
|
60
60
|
offset = -10
|
61
61
|
expect(uncap.seek(-10, IO::SEEK_END)).to eq(0)
|
@@ -70,19 +70,19 @@ describe ZipTricks::RemoteIO do
|
|
70
70
|
@buf.binmode
|
71
71
|
5.times { @buf << Random.new.bytes(1024 * 1024 * 3) }
|
72
72
|
@buf.rewind
|
73
|
-
|
73
|
+
|
74
74
|
@subject = described_class.new
|
75
|
-
|
75
|
+
|
76
76
|
allow(@subject).to receive(:request_object_size).and_return(@buf.size)
|
77
77
|
allow(@subject).to receive(:request_range) {|range|
|
78
78
|
@buf.read[range].tap { @buf.rewind }
|
79
79
|
}
|
80
80
|
end
|
81
|
-
|
81
|
+
|
82
82
|
after :each do
|
83
83
|
@buf.close; @buf.unlink
|
84
84
|
end
|
85
|
-
|
85
|
+
|
86
86
|
context 'without arguments' do
|
87
87
|
it 'reads the entire buffer and alters the position pointer' do
|
88
88
|
expect(@subject.pos).to eq(0)
|
@@ -93,6 +93,14 @@ describe ZipTricks::RemoteIO do
|
|
93
93
|
end
|
94
94
|
|
95
95
|
context 'with length' do
|
96
|
+
it 'supports an unlimited number of reads of size 0 and does not perform remote fetches for them' do
|
97
|
+
expect(@subject).not_to receive(:request_range)
|
98
|
+
20.times do
|
99
|
+
data = @subject.read(0)
|
100
|
+
expect(data).to eq('')
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
96
104
|
it 'returns exact amount of bytes at the start of the buffer' do
|
97
105
|
bytes_read = @subject.read(10)
|
98
106
|
expect(@subject.pos).to eq(10)
|
@@ -102,10 +110,10 @@ describe ZipTricks::RemoteIO do
|
|
102
110
|
|
103
111
|
it 'returns exact amount of bytes from the middle of the buffer' do
|
104
112
|
@subject.seek(456, IO::SEEK_SET)
|
105
|
-
|
113
|
+
|
106
114
|
bytes_read = @subject.read(10)
|
107
115
|
expect(@subject.pos).to eq(456+10)
|
108
|
-
|
116
|
+
|
109
117
|
@buf.seek(456)
|
110
118
|
expect(bytes_read).to eq(@buf.read(10))
|
111
119
|
end
|
@@ -113,11 +121,11 @@ describe ZipTricks::RemoteIO do
|
|
113
121
|
it 'returns the last N bytes it can read' do
|
114
122
|
at_end = @buf.size - 4
|
115
123
|
@subject.seek(at_end, IO::SEEK_SET)
|
116
|
-
|
124
|
+
|
117
125
|
expect(@subject.pos).to eq(15728636)
|
118
126
|
bytes_read = @subject.read(10)
|
119
127
|
expect(@subject.pos).to eq(@buf.size) # Should have moved the pos pointer to the end
|
120
|
-
|
128
|
+
|
121
129
|
expect(bytes_read.bytesize).to eq(4)
|
122
130
|
|
123
131
|
expect(@subject.pos).to eq(@buf.size)
|
@@ -6,16 +6,16 @@ describe ZipTricks::RemoteUncap, webmock: true do
|
|
6
6
|
after :each do
|
7
7
|
File.unlink('temp.zip') rescue Errno::ENOENT
|
8
8
|
end
|
9
|
-
|
9
|
+
|
10
10
|
it 'returns an array of remote entries that can be used to fetch the segments from within the ZIP' do
|
11
11
|
payload1 = Tempfile.new 'payload1'
|
12
12
|
payload1 << Random.new.bytes((1024 * 1024 * 5) + 10)
|
13
13
|
payload1.flush; payload1.rewind;
|
14
|
-
|
14
|
+
|
15
15
|
payload2 = Tempfile.new 'payload2'
|
16
16
|
payload2 << Random.new.bytes(1024 * 1024 * 3)
|
17
17
|
payload2.flush; payload2.rewind
|
18
|
-
|
18
|
+
|
19
19
|
payload1_crc = Zlib.crc32(payload1.read).tap { payload1.rewind }
|
20
20
|
payload2_crc = Zlib.crc32(payload2.read).tap { payload2.rewind }
|
21
21
|
|
@@ -32,9 +32,9 @@ describe ZipTricks::RemoteUncap, webmock: true do
|
|
32
32
|
end
|
33
33
|
end
|
34
34
|
payload1.rewind; payload2.rewind
|
35
|
-
|
35
|
+
|
36
36
|
expect(File).to be_exist('temp.zip')
|
37
|
-
|
37
|
+
|
38
38
|
allow_any_instance_of(described_class).to receive(:request_object_size) {
|
39
39
|
File.size('temp.zip')
|
40
40
|
}
|
@@ -44,22 +44,22 @@ describe ZipTricks::RemoteUncap, webmock: true do
|
|
44
44
|
f.read(range.end - range.begin + 1)
|
45
45
|
end
|
46
46
|
}
|
47
|
-
|
47
|
+
|
48
48
|
payload1.rewind; payload2.rewind
|
49
|
-
|
49
|
+
|
50
50
|
files = described_class.files_within_zip_at('http://fake.example.com')
|
51
51
|
expect(files).to be_kind_of(Array)
|
52
52
|
expect(files.length).to eq(2)
|
53
|
-
|
53
|
+
|
54
54
|
first, second = *files
|
55
|
-
|
55
|
+
|
56
56
|
expect(first.name).to eq('first-file.bin')
|
57
57
|
expect(first.size_uncompressed).to eq(payload1.size)
|
58
58
|
File.open('temp.zip', 'rb') do |readback|
|
59
59
|
readback.seek(first.starts_at_offset, IO::SEEK_SET)
|
60
60
|
expect(readback.read(12)).to eq(payload1.read(12))
|
61
61
|
end
|
62
|
-
|
62
|
+
|
63
63
|
expect(second.name).to eq('second-file.bin')
|
64
64
|
expect(second.size_uncompressed).to eq(payload2.size)
|
65
65
|
File.open('temp.zip', 'rb') do |readback|
|
@@ -67,15 +67,15 @@ describe ZipTricks::RemoteUncap, webmock: true do
|
|
67
67
|
expect(readback.read(12)).to eq(payload2.read(12))
|
68
68
|
end
|
69
69
|
end
|
70
|
-
|
70
|
+
|
71
71
|
it 'can cope with an empty file within the zip' do
|
72
72
|
payload1 = Tempfile.new 'payload1'
|
73
73
|
payload1.flush; payload1.rewind;
|
74
|
-
|
74
|
+
|
75
75
|
payload2 = Tempfile.new 'payload2'
|
76
76
|
payload2 << Random.new.bytes(1024)
|
77
77
|
payload2.flush; payload2.rewind
|
78
|
-
|
78
|
+
|
79
79
|
payload1_crc = Zlib.crc32(payload1.read).tap { payload1.rewind }
|
80
80
|
payload2_crc = Zlib.crc32(payload2.read).tap { payload2.rewind }
|
81
81
|
|
@@ -90,9 +90,9 @@ describe ZipTricks::RemoteUncap, webmock: true do
|
|
90
90
|
end
|
91
91
|
end
|
92
92
|
payload1.rewind; payload2.rewind
|
93
|
-
|
93
|
+
|
94
94
|
expect(File).to be_exist('temp.zip')
|
95
|
-
|
95
|
+
|
96
96
|
allow_any_instance_of(described_class).to receive(:request_object_size) {
|
97
97
|
File.size('temp.zip')
|
98
98
|
}
|
@@ -102,22 +102,22 @@ describe ZipTricks::RemoteUncap, webmock: true do
|
|
102
102
|
f.read(range.end - range.begin + 1)
|
103
103
|
end
|
104
104
|
}
|
105
|
-
|
105
|
+
|
106
106
|
payload1.rewind; payload2.rewind
|
107
|
-
|
107
|
+
|
108
108
|
files = described_class.files_within_zip_at('http://fake.example.com')
|
109
109
|
expect(files).to be_kind_of(Array)
|
110
110
|
expect(files.length).to eq(2)
|
111
|
-
|
111
|
+
|
112
112
|
first, second = *files
|
113
|
-
|
113
|
+
|
114
114
|
expect(first.name).to eq('first-file.bin')
|
115
115
|
expect(first.size_uncompressed).to eq(payload1.size)
|
116
116
|
File.open('temp.zip', 'rb') do |readback|
|
117
117
|
readback.seek(first.starts_at_offset, IO::SEEK_SET)
|
118
118
|
expect(readback.read(0)).to eq(payload1.read(0))
|
119
119
|
end
|
120
|
-
|
120
|
+
|
121
121
|
expect(second.name).to eq('second-file.bin')
|
122
122
|
expect(second.size_uncompressed).to eq(payload2.size)
|
123
123
|
File.open('temp.zip', 'rb') do |readback|
|
@@ -6,17 +6,17 @@ describe ZipTricks::StoredSizeEstimator do
|
|
6
6
|
raw_file_1 = SecureRandom.random_bytes(1024 * 20)
|
7
7
|
raw_file_2 = SecureRandom.random_bytes(1024 * 128)
|
8
8
|
raw_file_3 = SecureRandom.random_bytes(1258695)
|
9
|
-
|
9
|
+
|
10
10
|
predicted_size = described_class.perform_fake_archiving do | estimator |
|
11
11
|
r = estimator.add_stored_entry("first-file.bin", raw_file_1.size)
|
12
12
|
expect(r).to eq(estimator), "add_stored_entry should return self"
|
13
|
-
|
13
|
+
|
14
14
|
estimator.add_stored_entry("second-file.bin", raw_file_2.size)
|
15
|
-
|
15
|
+
|
16
16
|
r = estimator.add_compressed_entry("second-file.bin", raw_file_2.size, raw_file_3.size)
|
17
17
|
expect(r).to eq(estimator), "add_compressed_entry should return self"
|
18
18
|
end
|
19
|
-
|
19
|
+
|
20
20
|
expect(predicted_size).to eq(1410524)
|
21
21
|
end
|
22
22
|
end
|
@@ -7,29 +7,29 @@ describe ZipTricks::StreamCRC32 do
|
|
7
7
|
via_from_io = described_class.from_io(raw)
|
8
8
|
expect(via_from_io).to eq(crc)
|
9
9
|
end
|
10
|
-
|
10
|
+
|
11
11
|
it 'allows in-place updates' do
|
12
12
|
raw = StringIO.new(SecureRandom.random_bytes(45 * 1024 * 1024))
|
13
13
|
crc = Zlib.crc32(raw.string)
|
14
|
-
|
14
|
+
|
15
15
|
stream_crc = described_class.new
|
16
16
|
stream_crc << raw.read(1024 * 64) until raw.eof?
|
17
17
|
expect(stream_crc.to_i).to eq(crc)
|
18
18
|
end
|
19
|
-
|
19
|
+
|
20
20
|
it 'supports chained shovel' do
|
21
21
|
str = 'abcdef'
|
22
22
|
crc = Zlib.crc32(str)
|
23
|
-
|
23
|
+
|
24
24
|
stream_crc = described_class.new
|
25
25
|
stream_crc << 'a' << 'b' << 'c' << 'd' << 'e' << 'f'
|
26
|
-
|
26
|
+
|
27
27
|
expect(stream_crc.to_i).to eq(crc)
|
28
28
|
end
|
29
|
-
|
29
|
+
|
30
30
|
it 'allows in-place update with a known value' do
|
31
31
|
crc = Zlib.crc32
|
32
|
-
|
32
|
+
|
33
33
|
stream_crc = described_class.new
|
34
34
|
stream_crc << "This is some data"
|
35
35
|
stream_crc.append(45678, 12910)
|